{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Unit + MSE Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 30)\n",
    "        self.fc2 = nn.Linear(30, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.fc1(x)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=2000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        #print('Predicted: ', predicted)\n",
    "        #print('target: ', t.Tensor(labels).int())\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, vector_labels.float())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net1 = Net()\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(net1.parameters(), lr = 0.01)\n",
    "loss_scores, train_scores, test_scores = fit(net1, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xV9f3H8dcnOwRCAmElIWHvTSC4cIGiCKg/B07EQXF0Wdv6a3loq/XXVlurrcoQoUpFBEcFRS2OuhMSAoQNIZANScgie9zv749z0RACuUnuzU1uPs/HI4/ce873nPM5nPDOzRnfrxhjUEop5bm83F2AUkop19KgV0opD6dBr5RSHk6DXimlPJwGvVJKeTgfdxfQUFhYmBkwYIC7y1BKqQ5l27Zt+caYXo3Na3dBP2DAABITE91dhlJKdSgikna2eXrqRimlPJwGvVJKeTgNeqWU8nAa9Eop5eE06JVSysNp0CullIfToFdKKQ/X7u6jV0qpNnXiMCS/Ce2hy/bgcIhZ6PTVatArpTq3zb+Ew58C4u5KIDJGg14ppZwq74AV8pctgem/dHc1LqPn6JVSnVf8MvD2h8nO/xTdnmjQK6U6p/IC2LkOxt0IQWHurgabzVBWVeuSdWvQK6U6p6TXoKYcYu93dyWUVNZw32uJ3P96Ejab8y8Ka9ArpTqfulrY+jIMuAj6jnFrKSm5pVz74jd8cTCPmSN7Iy64JqwXY5VSncPxPXD4c+t14VEoyYSrn3FbOcYYPtp9jF+9lYy/rxev3xtL7KCeLtmWBr1SyvPZbLBhIeQf+GFa71Ew7Eq3lHPw+EmefH8vXx3KZ1xkd5bdPpnwkECXbU+DXinl+VI/s0J+7j9g1LXWNN8u4OXdpmWUV9fy9EcHWBOXRld/Hx6fM4rbp0Xj6+3as+ga9Eopzxe3DLr2gXHzwcfPLSUkpRfyi/U7OXqijNtjo3l45jBCg9qmFg16pZRnyzsIKVvgkt+4JeSNMfz90xSe//Qg/boHsvbeaZw32DXn4s9Gg14p5dm2LgdvP4i52y2bX/5lKn/75CDXTgjnyWvH0C3At81r0KBXSnmuikLYsRbG3ghde7X55jfuzOZPH+5nzvhwnr1pAl5e7ulPR++jV0p5rqQ19oeiFrf5prceKeCR9TuZMiCUZ24Y57aQB/1Er5TyVKceioq+EPqNa7PNVtfaeO27ozz3ySEiQwNZcUcMAb5te3dPQxr0SinPdOADKE6HWf/XZpv8fH8uT76/l9T8MqYP68Ufrx/bZnfWnIsGvVLKM8UthZAoGH61yzdVXFHD7zfu4Z3tWQzqFcTqu6Zw6YjeLt+uozTolVKeJ3s7pH8HVzzl8oeivknJ55cbdnL8ZBU/vXwoD146BD+f9nX506FqRGSWiBwQkRQRebSR+YtFZJeI7BCRr0VkVL15/2tf7oCIuOd5Y6VU5xK3DPy6wqQ7XLqZ95Ozuf2VeAL8vHnn/vP5+cxh7S7kwYFP9CLiDbwIzAQygQQR2WiM2Vuv2VpjzDJ7+7nAs8Ase+DPB0YD4cAnIjLMGFPn5P1QSinLyeOw+21rSL6A7i7bTOLRAh5ev5PJUaGsuSeWQD/3XnA9F0dO3UwFUowxqQAisg6YB3wf9MaYknrtg4BTHSrPA9YZY6qAIyKSYl/fd06oXSnliYyxhverKm3Z8oe2gK3GpbdUpuaVcu9riUSEBPLynTHtOuTBsaCPADLqvc8EYhs2EpEHgYcBP+CyesvGNVg2opFlFwGLAKKiohypWynlqQ5shnW3tm4dw2dDz8HOqaceYwyJaYU8smEnXiL8c+GUdnFXTVMcCfrG7vI/YwgUY8yLwIsiciuwBFjQjGVXACsAYmJinD+8ilKq44hbCt37w61v0niEOKDHQKeWVFNn49N9x1n+ZSrb04voEeTHygUxRPcMcup2XMWRoM8E+td7Hwlkn6P9OmBpC5dVSnVmOclw9CuY+QT0Ge3WUowxbD1SwMad2Xy4+xgFZdVE9ejCk/NGc8Pk/u3+dE19jgR9AjBURAYCWVgXV0/7u0pEhhpjDtnfzgZOvd4IrBWRZ7Euxg4FtjqjcKWUB4pfbvUTP+lOt5ZRU2fj0bd38XZSJoG+3swY1Yc54/px2Yje+Li473hXaDLojTG1IvIQ8DHgDawyxuwRkSeARGPMRuAhEZkB1ACFWKdtsLdbj3XhthZ4UO+4UUo1qjQPdq2HiXdAYKjbyqisqePB15P4dH8uP75sCPdfMpgufh37kSOHqjfGbAY2N5j2WL3XPz3Hsk8BT7W0QKVUJ7FtNdRVu6UDslPyS6t44F9JJKQV8OS1Y7hjWrTbanGmjv1rSinVcdnqrKdX62oAAwkrYcgM6DWszUtJO1HGyq+OsD4xA5sx/H3+ROaMD2/zOlxFg14p5R4JK+HDX50+7bxlbVrC8ZJKnvn4AO8kZeLj5cV1EyO4b/oghvTu2qZ1uJoGvVKq7dnqIO4liJhs9UcD4Bfk0u6Eq2ttZBaWA9Y93h/tPsaLn6dQW2e458KB3HfRIHoHB7hs++6kQa+UansHP4bCo3D54xB9nss3d+j4SRat2caR/LLTpl85ug+/uXpkh7kfvqU06JVSbS/uJQiOhJFzXb6pj/cc4+E3dxDo580frx9LF/v97/17dGFSlPvu7mlLGvRKqbZ1bLf1UNSM34O36yLoeEkly79IZdU3RxgX2Z3ld0ymX/dAl22vPdOgV0q1rfil4BPosoeiDueVsvyLw7y7PYs6m+GWqVE8PmeU24fzcycNeqVU2ynLh+QNMPE26NLDqasuLKvmuU8O8q/4dHy9hflTorj3ooEef/7dERr0Sqm2s2011FU55aGo4yWVHM0vI6e4ktS8Ul79Lo2TlTXcGhvFz2YMI6yrvxMK9gwa9EqptlFbDVtXwuDLodfw5i9eZ+OdpCy+SsknKa2QrKKK0+ZfOCSMJdeMZETfYGdV7DE06JVSbWPvv6H0GMx7odmLHs0v4+H1O0hKL6JvcACTB4Ryz4UDGdqnK/26BxIeEtDh+6NxJf2XUUq5njHWLZU9h1qf6B1ezPB6fDpPfbAPX2/h+fkTmDs+HJEW9lPfSWnQK6VcL2MrZG+Hq/8CXo5182uzGX6/aQ+vfpfGRUPDeOaG8fTt7plPrrqaBr1SyvXil1oDdY+/xaHm1bU2Htmwk407s7n3woH85uqReHnpp/iW0qBXSrVcTQUUZ527TUUB7N0I5z0A/k13FnaitIqH1+/ki4N5/HrWCBZfPEhP1bSSBr1SquXW3gxHvmi6nXjD1EXnbFJda+PVb4/y908PUVFTx5+uH8v8qVFOKrRz06BXSrVMzk4r5CcvhOgLzt02pD+EnD2092aX8NDaJFLzy7hkeC+WzB7lcV0Fu5MGvVKqZeKWgW8QzPgdBIa0eDVbjxRwz6sJBPn5sHrhFC4d3ttpJSqLBr1SqvlKc2H3WzBpQatC/pO9x3lwbRIRoYGsuSeWiJDO2emYq2nQK6WaL3GVfXzXHzVrsX05JayNTyerqILsogoO5ZYyOjyY1XdNoad2WeAyGvRKqeaprbKGARx6BYQNdWiROpth+ZeH+duWg/h6ezGgZxCRoYFcPLwXP75sKF39NYpcSf91lVI/qC6H6tJzt9m3CcryHOqYrKi8mqT0Ql78/DDb0gq5emxf/nDtWHoE+TmpYOUIDXqllKWiEJ6fAJVFTbcNGw6DL2t0ljGGdQkZrPr6CIdyrV8awQE+PHfzBOZN0O4L3EGDXillSXrNCvkZvwP/buduO+AiaCSwq2rrePy9PaxLyGBiVAiPXDGMydE9mNA/hEC/zjvwh7tp0CuloK4W4ldYAX7hz1u0ioyCcn6ybjvb04t46NIhPDxzmHZb0E5o0CulYP8mKMmEq59u1mJF5dVs2pnNpp05bD1aQBc/b5beNomrxvZzUaGqJTTolVLWw0+hA2DYLIcX2ZZWyOJ/bSPvZBVDe3flFzOHcd2kCCJDu7iuTtUiDgW9iMwCnge8gZXGmD81mP8wcC9QC+QBdxtj0uzz6oBd9qbpxpi5TqpdKeUMWUmQEQdX/hG8HDuP/sbWdB57bzfhIYH8+8ELmNC/5Q9NKddrMuhFxBt4EZgJZAIJIrLRGLO3XrPtQIwxplxE7geeBm62z6swxkxwct1KqfpqqwHTsmXjloJfN5h4e5NNU3JL+cdnh3hvRzbTh/XiH/Mn0r2Lb8u2q9qMI5/opwIpxphUABFZB8wDvg96Y8zn9drHAU3/xCilnCNxFbzfsguo34tdDAGNj7VaVlXLtrRC1sSlsWXvcfx9vPjxZUP42YxheOvF1g7BkaCPADLqvc8EYs/R/h7gw3rvA0QkEeu0zp+MMf9uuICILAIWAURFabekSjnMVgdf/w16j4ax/9OydXj5wMQ7TptUXF7Diq8O88XBPPblnKTOZgjp4stPLh/KgvOitbuCDsaRoG/sV3ajfyOKyO1ADHBxvclRxphsERkEfCYiu4wxh09bmTErgBUAMTExLfz7U6lO6MCHUJQON62BUa2//FVbZ2Pt1nSe3XKQkooapg7swQOXDGZSdCixA3voANwdlCNHLRPoX+99JJDdsJGIzAB+C1xsjKk6Nd0Yk23/nioi/wUmAocbLq+UaoG4pdA9CkbMbtVq0k+Usyk5m7e3ZZKaX8a0QT147JrRjApv/HSO6lgcCfoEYKiIDASygPnArfUbiMhEYDkwyxiTW296KFBujKkSkTDgAqwLtUqp1spJhrSvYeaTDt8tU9+x4kreT85mU3IOOzOsbg8mRYWw7PZJXDm6r3ZV4EGaDHpjTK2IPAR8jHV75SpjzB4ReQJINMZsBJ4BugIb7D8cp26jHAksFxEb4IV1jn5voxtSSjVPvH3gj0l3NN0WKKmsISmtkG1phcSlniAxrRBjYHR4MP971Qhmj+un98B7KIdOuBljNgObG0x7rN7rGWdZ7ltgbGsKVEo1ojQPdm2ASXdCYOhZmxWX1/DRnhw27czh28P52Ax4CYwKD+Znlw9jzvh+DOqlQ/Z5Or2yolRH9P3AH2d2FVxWVcsn+46zaWc2XxzMo6bOEN2zC4svHswFQ8KY0D+EIO3/vVPRo61UR1NbBYmvwJAZpw38cbKyhhc+T+HVb49SWWOjb3AAd543gLnjwxkX2V3PuXdiGvRKdTR73oXS4zDtJcAavWlDYgZ/+c8B8kuruW5iBLdMjSImOlR7j1SABr1SHYsxEPeSfeCPyykqr+bHb2znq0P5xESHsuquKYyL1H5n1Ok06JXqSNLjIGcnzH6W/cdPsui1bRwrruSp68Zw69QoPT2jGqVBr1QHkVFQjnz0LGE+wTx1dAxvb/qWrv4+vLFoGpOjz37njVIa9Eq1c7uziln+ZSrbk3fyhd8WXq67hg8PlDBtUE/+eP1Y+gQHuLtE1c5p0CvVTpVV1fLrt5N5PzmHrv4+vBy9Fa/jXtz146dYHBbt7vJUB6JBr1Q7dDS/jEVrEknJLeWnlw/lntjeBL90H4ycQ4CGvGomDXql2hGbzfDxnmP8+u1kvLyE1+6O5cKhYZCwEiqLYdoD7i5RdUAa9Eq1A1W1dby3I5uXv0zlUG4pI/p2Y8UdMUT17AI2G8Qvh/CJ0H+qu0tVHZAGvVJuZIzhg105/HHzfrKKKhjRtxvP3TyB2eP64evtZTU6/BnkH4TrXwa9fVK1gAa9Um6yO6uYJzbtZevRAkb2C+ap68Zw8bBeZ94LH78UuvaFUde6p1DV4WnQK9XGck9W8pePD7BhWyY9uvjxx+vHclNM/8bHX807ACmfwKVLwMev7YtVHkGDXqk2klVUwdvbMln+xWGq62zce+FAfnz5UIIDfE9vuOVx2P++9bqyGLz9IWZh2xesPIYGvVIuVF1rY8O2DN5NyiIxrRCAGSP78NvZIxkYFnTmAkUZ8O0/oN846DHImjbwYggKa8OqlafRoFfKBYwxfLovl6c27+NIfhnD+3TjkSuGMWd8ONE9Gwn4UxJetr7ftAZC+p+9nVLNoEGvlJMVlVfzk3U7+PJgHoN7BbF64RQuHd676QWry2DbP2HkNRryyqk06JVyopLKGu5ctZX9OSd57JpR3HFe9A+3STZl5zp9KEq5hAa9Uk5SVlXLwtUJ7M0uYdntk5kxqo/jC9ts1mDf/SZA/1jXFak6JQc/aiilzqW4ooZ7X01kR0YR/7hlYvNCHiDV/lDUtAf0oSjldPqJXnVuiash6bWm23XrBzeuBh//M2Z9k5LPIxt2knuyimdvGs9VY/v9MLO2CjYshJM5515/cSZ07QOjr2vmDijVNA161XlVl8Mnv4PAEOg59OztaivhwAfWWK3j538/uayqlmc+PsA/vz3KoF5BvHP/+Yzv32AYv91vW8sOuAh8ztFvfJeeMOFWfShKuYQGveq8ktdBZRHc8gZEn3/2dsbAi7HWWK3jbsZm4J3tWTz90X5yT1Zx1/kD+PWsEQT6eZ+5XNxL0GskLNikp2SU22jQq87JGIhbBv3GQ9R5524rQt3UH+G9+WE++OBdlh/tTXJmMeP7h7D09slnH8Yv7Vs4tgvmPK8hr9xKg151Toc/g/wDcO2yRkO4orqO+CMn2JZWSOLRQvZnhPG5VxASv5Ti4N/w7E3juXZCBF6N9U9zStxLEBgKY29y4Y4o1TSHgl5EZgHPA97ASmPMnxrMfxi4F6gF8oC7jTFp9nkLgCX2pn8wxrzqpNqVarn4ZRDUG8Zc//2k6lobXxzMY9PObD7Zd5zy6jq8vYRR/YKZN2UIeSdv4aqUVVx932AIiTz3+guPwoHNcMHPwK+La/dFqSY0GfQi4g28CMwEMoEEEdlojNlbr9l2IMYYUy4i9wNPAzeLSA/gcSAGMMA2+7KFzt4RpRyWfwgO/Qcu+V/w8aeksoY34tNZ9c0RjpdUEdrFl2snRnDVmL5MigolyN/+36T45/Dcati6Aq74w7m3sfVlQGDKvS7fHaWa4sgn+qlAijEmFUBE1gHzgO+D3hjzeb32ccDt9tdXAluMMQX2ZbcAs4A3Wl+6ard2vGFd6Dylfyxc+hvnrb+mEjY+BGV5LVu+OAu8/cgbfisrP9zH2rh0TlbVcv7gnvzfdWOZPqxX40+zdo+EUXMhYZV17v1cMhJg9LXQPaJlNSrlRI4EfQSQUe99JnCuR/fuAT48x7Jn/OSLyCJgEUBUVJQDJal27du/Q2ku9BwM5QWQ+mcYeyOEneMWxubYtcH6Cp8I3s2/HbHCtzufhd3Nz17YTZ3NcNXYfiyePpixkd2bXnj6L6E0D2oqzt0ufCJc9Eiza1PKFRwJ+sauNplGG4rcjnWa5uLmLGuMWQGsAIiJiWl03aqDqCyB3H3WaZFLfm2F4t9GW+fEZ/+19es3BuKWQp8xcN/nzbqbpai8muc+OcSauDR8vYVbp/bnngsHWeOyOqrPaFj4QQsKV8p9HAn6TKB+V3qRQHbDRiIyA/gtcLExpqrespc0WPa/LSlUdRDZSYCByBjrfdde1qf5HWvhsiXWXSitceRLyN0Dc19oVshvSMzgqc37KKmo4ZapUTw8cxg9u575lKtSnsiRvm4SgKEiMlBE/ID5wMb6DURkIrAcmGuMya0362PgChEJFZFQ4Ar7NOWpMhOs7xGTf5g2bTHUlEPSmtavP36Z9RTp2Bsdam6M4dktB/nlW8kM69OND35yEU9dN1ZDXnUqTQa9MaYWeAgroPcB640xe0TkCRGZa2/2DNAV2CAiO0Rko33ZAuBJrF8WCcATpy7MKg+VuQ3ChlndCpzSd6zVBcDWFVBX2/J1F6TCgQ8h5m7wPUd3AnZ1NsNj7+3h758e4sbJkay9N5aR/YJbvn2lOiiH7qM3xmwGNjeY9li91zPOsewqYFVLC1QdiDHWJ/phV545L3YxvHmb1e/LqHktW3/8CvDygZh7mmxaXFHDo28n8+HuY/xo+iAevWoEok+nqk5Kn4xVzlN4FMrzfzg/X9/wqyAkGj5eArveatn6Uz61encM7nfOZl8dyuOXG5LJK61iyeyR3HvRoJZtTykPoUGvnCdrm/U9opGg9/KGyx+Dr/5qPbDUEmFD4MKfn33zRRW8+HkKa+PTGdwriBV3ns+4yJCztleqs9CgV86TmQC+XaD3qMbnj73B+nKy/cdKWP5FKpt2ZmOAuy8YyK9mDSfA17vJZZXqDDTolfNkJkD4JPBumx+r3JOVPPPRAd5KyiTQ15sF5w/g7gsHEhES2CbbV6qj0KBXzlFTCTnJcN6DLttEUXk1GQUVZBdXsCermFe+PkJ1nY1FFw3i/ksGE9JFB+1QqjEa9Mo5ju0CW03jF2JbaUdGESu+PMxHu49hq/fc9MxRffjt1SMZEBbk9G0q5Uk06JVzfP+glPOCfv+xEh5/bw/xRwoIDvDhvumDmNg/lPCQACJCAvWhJ6UcpEGvnCMzAbr3b/LWR0fU2Qwrv0rlr/85SHCgD0tmj2T+1Ci6+uuPq1Itof9zlHNkJp7e7UEL5ZZU8tDa7Ww9WsCs0X156rox+sldqVbSoFetd/I4FKdD7I9atZr80ipuXRlPdlEFz940nusmRujTrEo5gQa9ar2sROt75JQWr6KovJrbV8aTWVjOqwunEjuop5OKU0pp0KvWy0wAL1/oN65FixeVV3PHK1tJzS/jlQUxGvJKOZkGvWq9zEToOwZ8m/+g0ql+aU6UVbHs9slcNLSXCwpUqnPToFetY6uDrCSYeNtZmxhjKK6oIbuokqLy6u+nf7TnGK99l8aQ3l15+c4Yx4byU0o1mwa9ap3cfVBTdsb5+aLyaj7afYxNydlsTy+ivLrujEVF4N4LB/LIldovjVKupEGvWufUg1L2J2Jr62w8+f5eXo9Pp9ZmGNCzCzfF9CcyNJDwkEB6BPl9P5BwWDd/Bvfq6p66lepENOhV62QmWkP7hQ6ksqaOn7yxnf/sPc4tU6O4LTaK0eHBeoukUm6mQa9aJysRImI4WVXLfa8lEpdawONzRrHwgoHurkwpZadBr1quogjy9lM8eC43L/uOlNxSnp8/gXkTItxdmVKqHg161XLZSQD8Kt6fbFPBqrumMH2Y3h6pVHujQa8cdrKyhpziSrKLKsguqqTvjs1cYoTjwaPYeOeF2l2wUu2UBr06p1NdBe/NKeFkZS1Xem1luGQCMNk7nuMBUfzrwSu0Z0ml2jH936ka1bCr4OsnRjAsoIhb4v6Bl6l3T/zUh0FDXql2Tf+HqtOUVNawZc9x/hWfxvb0Iq4c3Yf/u26s1VXwlsesRj9NtvqeB/Dycl+xSimHaNB3cjV1NvZml5CYVkhc6gm+OJhHda2NiJBA/nrjeK6fZO8quLoMtv0TRs6B0Gh3l62UagYN+k7svR1ZLHl3NyeragGICAnk1qlRzBkfzqSokNMfdNq5DiqLYdr9bqpWKdVSDgW9iMwCnge8gZXGmD81mD8deA4YB8w3xrxVb14dsMv+Nt0YM9cZhauWq62z8eeP9vPyV0eYMiCUu84fyOToUPp2D2h8AZsN4pdB+EToH9u2xSqlWq3JoBcRb+BFYCaQCSSIyEZjzN56zdKBu4BHGllFhTFmghNqVU5QXF7Dg2uT+DolnwXnRbPkmlH4ejdxnj31M8g/CNetsHoiU0p1KI58op8KpBhjUgFEZB0wD/g+6I0xR+3zbC6oUTnJ8ZJKFqzaSmpeGU/fMI6bYvr/MNMYSH7Tetq1oV3roWsfGH1d2xWrlHIaR4I+Asio9z4TaM7f7wEikgjUAn8yxvy7YQMRWQQsAoiKimrGqpWjjuaXcfsr8RSWVbN64RQuGBJ2eoODH8O75xjzdeYT4OPn2iKVUi7hSNA39re6acY2oowx2SIyCPhMRHYZYw6ftjJjVgArAGJiYpqzbtWEE6VVbN59jOc/OYjNwBuLpjEuMuTMhnEvQbdwWPwVSINTOeIFgY0so5TqEBwJ+kyg3t/4RALZjm7AGJNt/54qIv8FJgKHz7mQarbaOhsf7MohLvUExv6rMquogm8Pn6DOZhjRtxsv3DqJIb0b6f/9+F448gVc/jgEhZ05XynVoTkS9AnAUBEZCGQB84FbHVm5iIQC5caYKhEJAy4Anm5psepMpVW1bEjMYOVXR8gqqiCkiy/+PtYn8m4Bvvxo+iDmTghneJ9uZ+8XPn4p+ATC5LvarnClVJtpMuiNMbUi8hDwMdbtlauMMXtE5Akg0RizUUSmAO8CocAcEfm9MWY0MBJYbr9I64V1jn7vWTalHFRZU8fn+3PZlJzNp/tyqaq1MWVAKL+bO5rLR/TGy6sZd8aUnYDk9TB+PnTp4bqilVJu49B99MaYzcDmBtMeq/c6AeuUTsPlvgXGtrJGVU/uyUoWrEpgX04JYV39mD+lP9dOjGBiVGjLVrhtNdRWQuxi5xaqlGo39MnYDiTtRBl3vLKV/NIqXrptEleM6oNPU/fAn0tdDSSshEGXQu+RzitUKdWuaNB3ELuzirlrdQK1Nhuv3xvb8k/w9e19D07mwJznW78upVS7pUHfzhWVV/PcJ4dYE5dG727+rFt0HkN6d3POyuOWQo/BMGSmc9anlGqXNOjbqZziCt7bkc2yLw5TUlHD/KlR/GLmMKu7YGfISLAG9r7qGe1qWCkPp0HfjlTW1PHu9izeTcpi69ECAC4Y0pMls0cxsl+wczcWvxT8g2HCLc5dr1Kq3dGgbweKyqtZ810ar353lPzSaob27sovZg7jmvHhDHTFOKzFWdb5+djF4O+k00BKqXZLg74NGWNOe2gpo6CcV74+wvrEDMqr67h0eC8WTR/MtEE9zv5wkzMkrARjg6n3uW4bSql2Q4O+DaSfKGfl16m8tS0TPx8v+nUPpHugDwlHCxFg7oRwFk0fxIi+Tj4905jqcmukqOFXQ+gA129PKeV2GvQudCS/jL/+5wCbd+Xg7SVcMy6cIH9vcooqOX6ykrsvGMDCCwYSHhLYdkXtWg8VBTDtgbbbplLKrTTonWBnRhH/PZDH8L5dmRzdA39fL174LIXV3xzB19uL+6YPYuH5A88+glNbMQbilkHfsRB9vntrUUq1GQ36Fqqts/HVoXyWf3mYuNSC0+b5+82paG8AAAxhSURBVHhRXWfjhkmR/PLK4fQOdnPAn5L6X8jbB9cu1ZGilOpENOjPIae4gtXfHCU5s4i+wQGEhwTi7SUkpReyI72Isuo6+nUPYMnskVw/KZIj+WVsSysg7UQ586dEMTayu7t34XTxyyCoF4z5H3dXopRqQxr0dim5J9mRUQxYd8fEpRbw3o4sDDAmPJiEo4UcL8nBZgwj+wVzw+RIYgf1ZMbIPvjZuwXuEeTH5GgndE3gCicOw8GP4OJHwcdJD10ppTqETh/0tXU2ln+ZynOfHKSm7ofBrQJ9vbl9WjT3XDiQ/j26AFBnM9TU2Qjw9XZXuS0Xvwy8fCHmbndXopRqY5066I/kl/GL9TtISi9i9th+/HzmsO8H7QgN8qOr/+n/PN5egrdXBwz5ymLYsRbG3gDd+ri7GqVUG+uUQV9aVcsLn6Ww6usjBPh68fz8CcwdH+7ah5Tcafu/oLpU+5xXqpPy+KCvsxnyTlaRXVxBdlEFaSfKWf3NUfJLq7h+UgS/njWCPu3lrhhXsNVZp22izofwCe6uRinlBh4Z9GVVtfzqrWR2ZBRxvKSSWps5bf7k6FBWLohhQv8QN1XYhg58CEXpcMUf3F2JUspNPC7oa+tsPLQ2iS8O5jFnfDiRoYH06x5IREgg/UIC7N0P+Lq7zLYTtxS6R8Hw2e6uRCnlJh4V9MYYHt+4h88P5PHUdWO4LTba3SW5V04ypH0NM58Eb4861EqpZvCoESeWf5nK6/HpLL54sIY8WOfmfYNg0h3urkQp5UYeE/QpuaX8+aP9XDOuH7+6cri7y3G/0jzYtcEaWCSwnT7EpZRqEx7z9/yQ3l1ZddcUzhvUEy8vD71NsjkSV0Fdtd5SqZTynKAHuHR4b3eX0D7UVkHiK9ag32FD3V2NUsrNPObUjapnz7tQehym6ad5pZQGvecxBuJegrDhMPhyd1ejlGoHHAp6EZklIgdEJEVEHm1k/nQRSRKRWhG5ocG8BSJyyP61wFmFq7NIj4OcnRD7I+1zXikFOBD0IuINvAhcBYwCbhGRUQ2apQN3AWsbLNsDeByIBaYCj4uI3gLiSvFLISAExs93dyVKqXbCkU/0U4EUY0yqMaYaWAfMq9/AGHPUGJMM2BoseyWwxRhTYIwpBLYAs5xQt2pMUTrs2wSTF4BfkLurUUq1E44EfQSQUe99pn2aI1qzrGqurS8DAlPuc3clSql2xJGgb+xEr2lkWouXFZFFIpIoIol5eXkOrlqdproMkl6FkXMgpL+7q1FKtSOOBH0mUD85IoFsB9fv0LLGmBXGmBhjTEyvXr0cXLU6zc43rAFGpj3g7kqUUu2MI0GfAAwVkYEi4gfMBzY6uP6PgStEJNR+EfYK+zTlTDYbxC2D8InQf6q7q1FKtTNNBr0xphZ4CCug9wHrjTF7ROQJEZkLICJTRCQTuBFYLiJ77MsWAE9i/bJIAJ6wT1POdPgzOHHI+jSvt1QqpRoQYxw93d42YmJiTGJiorvL6FjWXA/H98DPdoGPn7urUUq5gYhsM8bENDbPo/q66TRsNqgssl4XHoHDn8KlSzTklVKN0qDviN57wLr4eoq3P8QsdF89Sql2TYO+oynOhOT1MOIaGDjdmtZrBASFubcupVS7pUHf0Wx9GTAw648QEuXuapRSHYD2XtmRVJfBtn/aH4rSkFdKOUaDviNJftO6CBt7v7srUUp1IBr0HYUx1kNR/SZA1DR3V6OU6kA06NszY8BWZ32lfAr5B2Da/fpQlFKqWfRibHu25lpI/e8P77v2gdHXua0cpVTHpEHfXmUkWCE/5gboNdyaNuAi8PF3a1lKqY5Hg769insJ/LvDnOfBv6u7q1FKdWB6jr49Ks6Cve/BpDs05JVSraZB3x4l2B+KmrrI3ZUopTyABn17U11uPRQ1YjaERru7GqWUB9Cgb2+S34SKQn0oSinlNHoxtqX2b4Z3F4Ot1rnrra2EvuMg+nznrlcp1Wlp0LeEMfDl0xAQDKPmOX/9Y/5HH4pSSjmNBn1LZGyF7O0w+68w5V53V6OUUuek5+hbIn4pBHSH8be4uxKllGqSBn1zFWfC3o0waQH4Bbm7GqWUapIGfXOdGvhj6n3urkQppRyiQd8cp+5x14E/lFIdiF6Mbcw3z0Pc0jOn11XrwB9KqQ5Hg76hqpPw5V8gJBoiJp45PyRaB/5QSnUoGvQN7XgDqkpgznMQGePuapRSqtX0HH19NhvEL4PIKRrySimPoUFfX8oWKDgMsYvdXYlSSjmNQ0EvIrNE5ICIpIjIo43M9xeRN+3z40VkgH36ABGpEJEd9q9lzi3fyeJegm7hrunWQCml3KTJc/Qi4g28CMwEMoEEEdlojNlbr9k9QKExZoiIzAf+DNxsn3fYGDPByXU7X+4+a+i+yx8Db193V6OUUk7jyMXYqUCKMSYVQETWAfOA+kE/D/id/fVbwAsibdwrV3kBrL6q5ctXFIJPAExe6LyalFKqHXAk6COAjHrvM4HYs7UxxtSKSDHQ0z5voIhsB0qAJcaYrxpuQEQWAYsAoqJa+CCSl/cPg2i31ODLoUuP1q1DKaXaGUeCvrFP5sbBNjlAlDHmhIhMBv4tIqONMSWnNTRmBbACICYmpuG6HRPQHW56rUWLKqWUJ3PkYmwm0L/e+0gg+2xtRMQH6A4UGGOqjDEnAIwx24DDwLDWFq2UUspxjgR9AjBURAaKiB8wH9jYoM1GYIH99Q3AZ8YYIyK97BdzEZFBwFAg1TmlK6WUckSTp27s59wfAj4GvIFVxpg9IvIEkGiM2Qi8AqwRkRSgAOuXAcB04AkRqQXqgMXGmAJX7IhSSqnGiTEtOyXuKjExMSYxMdHdZSilVIciItuMMY0+0q9PxiqllIfToFdKKQ+nQa+UUh5Og14ppTxcu7sYKyJ5QForVhEG5DupnI6iM+4zdM797oz7DJ1zv5u7z9HGmF6NzWh3Qd9aIpJ4tivPnqoz7jN0zv3ujPsMnXO/nbnPeupGKaU8nAa9Ukp5OE8M+hXuLsANOuM+Q+fc7864z9A599tp++xx5+iVUkqdzhM/0SullKpHg14ppTycxwR9UwOYewoR6S8in4vIPhHZIyI/tU/vISJbROSQ/Xuou2t1NhHxFpHtIvK+/f1A+2D0h+yD0/u5u0ZnE5EQEXlLRPbbj/l5nn6sReTn9p/t3SLyhogEeOKxFpFVIpIrIrvrTWv02Irl7/Z8SxaRSc3ZlkcEfb0BzK8CRgG3iMgo91blMrXAL4wxI4FpwIP2fX0U+NQYMxT41P7e0/wU2Ffv/Z+Bv9n3uRBrkHpP8zzwkTFmBDAea/899liLSATwEyDGGDMGq2v0+Xjmsf4nMKvBtLMd26uwxvMYijXs6tLmbMgjgp56A5gbY6qBUwOYexxjTI4xJsn++iTWf/wIrP191d7sVeBa91ToGiISCcwGVtrfC3AZ1mD04Jn7HIw1psMrAMaYamNMER5+rLHGyQi0j1bXBWtIUo871saYL7HG76jvbMd2HvCascQBISLSz9FteUrQNzaAeYSbamkzIjIAmAjEA32MMTlg/TIAeruvMpd4DvgVYLO/7wkUGWNq7e898ZgPAvKA1fZTVitFJAgPPtbGmCzgL0A6VsAXA9vw/GN9ytmObasyzlOC3pEBzD2KiHQF3gZ+1nCwdU8jItcAufZxh7+f3EhTTzvmPsAkYKkxZiJQhgedpmmM/Zz0PGAgEA4EYZ22aMjTjnVTWvXz7ilB78gA5h5DRHyxQv51Y8w79snHT/0pZ/+e6676XOACYK6IHMU6LXcZ1if8EPuf9+CZxzwTyDTGxNvfv4UV/J58rGcAR4wxecaYGuAd4Hw8/1ifcrZj26qM85Sgd2QAc49gPzf9CrDPGPNsvVn1B2hfALzX1rW5ijHmf40xkcaYAVjH9jNjzG3A51iD0YOH7TOAMeYYkCEiw+2TLgf24sHHGuuUzTQR6WL/WT+1zx59rOs527HdCNxpv/tmGlB86hSPQ4wxHvEFXA0cBA4Dv3V3PS7czwux/mRLBnbYv67GOmf9KXDI/r2Hu2t10f5fArxvfz0I2AqkABsAf3fX54L9nQAk2o/3v4FQTz/WwO+B/cBuYA3g74nHGngD6zpEDdYn9nvOdmyxTt28aM+3XVh3JTm8Le0CQSmlPJynnLpRSil1Fhr0Sinl4TTolVLKw2nQK6WUh9OgV0opD6dBr5RSHk6DXimlPNz/A8pjHkL5ZjW7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Unit + Cross Entropy Cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit2(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net2.parameters(), lr = 0.01)\n",
    "loss_scores2, train_scores2, test_scores2 = fit2(net2, criterion, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xUdb7/8dcnCQkQCC0hkAIBqQGkJBQRVEAQsGABxa6Lencta7t3H+7u3Z+77rr3uqteG1gRy4pgWRVdERFFVGrohBp6CIQAIYSa9v39MeO9WZbIQMpJZt7Px2MemTlzzpnP93HymPec7ylfc84hIiKhJ8zrAkRExBsKABGREKUAEBEJUQoAEZEQpQAQEQlREV4XcCZiY2NdSkqK12WIiNQpS5cu3eecizt5ep0KgJSUFDIyMrwuQ0SkTjGz7aeari4gEZEQpQAQEQlRCgARkRClABARCVEKABGREKUAEBEJUQoAEZEQFfQBUFrmmL5kBzNX7/a6FBGRWqVOXQh2NsIM3lm0gwNHihjWNZ7IiKDPPBGRgAT9t6GZ8eDwTmTnH+P9pTu9LkdEpNYI+gAAuKhTHH3aNOWFr7M4XlzqdTkiIrVCSASAmfHwiM7sLjjO9CXaCxARgRAJAICB57SgX7vmTPxGewEiIhBCAWBmPDy8E3sLT/C3hae8MZ6ISEgJmQAA6N++BYM6xDJp7mYKjxd7XY6IiKdCKgAAfjWyMweOFPHqvC1elyIi4qmQC4Bzk5py6bmtee37reQVnvC6HBERz4RcAAD8+4jOFJWU8fzXm7wuRUTEMyEZAO1io7mubzJTF+1g+/4jXpcjIuKJkAwAgPuHdaReeBh/nbXB61JERDwRsgHQMqY+dwxux2erdrN8R77X5YiI1LiQDQCAn194DnGNo/jTP9bhnPO6HBGRGhXSARAdFcG/j+jE0u35fL56j9fliIjUqJAOAICxacl0adWY/5q5TreIEJGQEvIBEB5m/OelqWTnH+ON+du8LkdEpMaEfAAADOoYy7AuLXnh6yz2Fh73uhwRkRqhAPD77aVdOVFSyl++0GmhIhIaFAB+7eMaMWFQez5Yms0ynRYqIiFAAVDOfUM7EB8Txe9nZFJWptNCRSS4KQDKiY6K4NejurIqu4D3MjRymIgEt4ACwMxGmtkGM8sys0dO8f4FZrbMzErMbOxJ791qZpv8j1vLTZ/rX+cK/6Nl5ZtTeWN6JdA3pRl/mbWBg0eLvC5HRKTanDYAzCwcmAiMAlKB680s9aTZdgC3AVNPWrY58CjQH+gHPGpmzcrNcqNzrpf/sfesW1GFzIzHxnSn4FgxT+iAsIgEsUD2APoBWc65Lc65ImAaMKb8DM65bc65VUDZScteAsx2zh1wzuUDs4GRVVB3teraOoafnZ/Cu4t3sHT7Aa/LERGpFoEEQCJQvkM82z8tEKdbdoq/++d3ZmanWoGZ3WVmGWaWkZeXF+DHVt4DF3eidZP6/PajNZSUnpxrIiJ1XyABcKov5kBPkfmpZW90zvUABvsfN59qBc65V5xz6c659Li4uAA/tvKioyJ49PJurN9TqCuERSQoBRIA2UByuddJQE6A669wWefcLv/fQnzHDvoFuM4ac0m3eIZ1acnTszeSnX/U63JERKpUIAGwBOhoZu3MLBIYD8wIcP2zgBFm1sx/8HcEMMvMIswsFsDM6gGXAWvOvPzqZWb8YUw3AP7z4zW6ZbSIBJXTBoBzrgS4F9+X+TrgPedcppk9ZmZXAJhZXzPLBsYBL5tZpn/ZA8Af8YXIEuAx/7QofEGwClgB7AJerfLWVYGkZg35j0s6M3dDHp+sCHTHR0Sk9rO69Ks2PT3dZWRk1PjnlpY5xr40n237jvDVQxfSolFUjdcgInK2zGypcy795Om6EjgA4WHGE9ecy+ETJTz22VqvyxERqRIKgAB1im/M3Rd14JMVOcxem+t1OSIilaYAOAP3DOlAl1aN+c1Hq8k/ottEiEjdpgA4A5ERYTx1bU/yjxTx+08zvS5HRKRSFABnqFtCE+4b2pFPVuTwxZrdXpcjInLWFABn4e4h59AtIYbffrSGfYdPeF2OiMhZUQCchXrhvq6gwuMlPPLhal0gJiJ1kgLgLHVpFcOvRnbmq3W5TF+iwWNEpO5RAFTCz85vx8BzWvDYZ2vZtu+I1+WIiJwRBUAlhIUZT47rSUSY8cD0FbpttIjUKQqASkpo2oDHr+rBip0HeXbOJq/LEREJmAKgClzeM4GxaUm88E0W8zfv87ocEZGAKACqyB+u6Ea72GgenL6CA7pKWETqAAVAFYmOiuD563uTf6SY/3h/pU4NFZFaTwFQhbolNOE3o7swZ/1eJn+/1etyRER+kgKgit06MIVLusXz3zPXs3T7Aa/LERGpkAKgipkZfxnbk4SmDbh36nIdDxCRWksBUA2aNKjHpBv7sP9IEQ9MX0FZmY4HiEjtowCoJt0Tm/D7y7sxb2Mez32t6wNEpPZRAFSj6/slc02fJJ75ahNz1mkUMRGpXRQA1cjMePyq7nRLiOGB6SvYqvsFiUgtogCoZvXrhfPSTWlEhBn/9nYGR06UeF2SiAigAKgRyc0b8tz1vcnae5h/f3+lDgqLSK2gAKghgzvG8etRXZm5Zo8OCotIrRDhdQGh5I7B7Vi/p5BnvtpE5/jGjOrR2uuSRCSEaQ+gBv14ULh3m6Y89N5KMnMKvC5JREKYAqCG1a8Xzss3p9G0YT3ueDOD3EPHvS5JREKUAsADLRvX57Vb0yk4VsyEN5dwtEhnBolIzVMAeKRbQhNeuKE3a3MOcf+0FZTqzCARqWEKAA8N7RLP/7ssldlrc/nz5+u8LkdEQozOAvLYbee3Y9v+o0z+fisJTRswYVA7r0sSkRChAKgFfndZKnsKjvOnf6yldZP6jNbpoSJSA9QFVAuEhxnPjO9FWptmPDB9BYu27Pe6JBEJAQqAWqJ+vXBevSWd5GYNuOOtDNbtPuR1SSIS5BQAtUiz6EjemtCf6MgIbnl9MTv2H/W6JBEJYgEFgJmNNLMNZpZlZo+c4v0LzGyZmZWY2diT3rvVzDb5H7eWm55mZqv963zOzKzyzan7Eps24O0J/SguLeOmyYvYW6gLxUSkepw2AMwsHJgIjAJSgevNLPWk2XYAtwFTT1q2OfAo0B/oBzxqZs38b78I3AV09D9GnnUrgkzH+Ma8fltf8gpPcMvkxRw8qnGFRaTqBbIH0A/Ics5tcc4VAdOAMeVncM5tc86tAspOWvYSYLZz7oBzLh+YDYw0s9ZAjHNugXPOAW8BV1a2McGkT5tmvHJLGlvyjnDrlCUc1jgCIlLFAgmARGBnudfZ/mmBqGjZRP/z067TzO4yswwzy8jLywvwY4PD4I5xvHBDb9bsKmDCG0s4VlTqdUkiEkQCCYBT9c0Het+CipYNeJ3OuVecc+nOufS4uLgAPzZ4jOjWiqev7cnibQf4+d+WcqJEISAiVSOQAMgGksu9TgJyAlx/Rctm+5+fzTpDzpheifzXVT34dmMed/9tGUUlJ/e0iYicuUACYAnQ0czamVkkMB6YEeD6ZwEjzKyZ/+DvCGCWc243UGhmA/xn/9wCfHIW9YeM8f3a8KcruzNn/V7umbqM4lKFgIhUzmkDwDlXAtyL78t8HfCecy7TzB4zsysAzKyvmWUD44CXzSzTv+wB4I/4QmQJ8Jh/GsAvgNeALGAzMLNKWxaEbhrQlj9c0Y3Za3O5b+pyhYCIVIr5TsKpG9LT011GRobXZXhu8vdb+eNna7mkWzzPX9+HyAhdzyciFTOzpc659JOn65ujDpowqB2PXp7KrMxc7n5HB4ZF5OwoAOqo289vxx/HdOOrdXv5+dtLOV6sEBCRM6MAqMNuPi+FP1/Vg7kb8/jZG0s4oovFROQMKADquBv6t+GpcT1ZuGU/N09eRMGxYq9LEpE6QgEQBK7uk8SkG/uwelcBN7y6kH2HT3hdkojUAQqAIDGye2tevSWdzXmHGffSArLzdStpEflpCoAgclHnlvxtQn/2Hz7B2BcXsCm30OuSRKQWUwAEmfSU5kz/t/ModY5xLy9g6fZ8r0sSkVpKARCEuraO4cOfD6Rpg3rc+NpCvlqb63VJIlILKQCCVJsWDfngFwPpFN+Yu97OYNriHV6XJCK1jAIgiMU2iuLdOwdwQac4Hvn7ap76cgN16dYfIlK9FABBLjoqgldvSee69GSe/zqLB6ev0K0jRASACK8LkOpXLzyM/76mB21aNOSvszawu+A4L9+cRtOGkV6XJiIe0h5AiDAz7hnSgWfH92L5joNcPWk+W/cd8bosEfGQAiDEjOmVyDt39ufgsWKunPgDCzbv97okEfGIAiAE9U1pzsd3n0/LxlHcPHkRUxfpDCGRUKQACFFtWjTkw7sHMrBDLL/5aDWPfrJGI4yJhBgFQAiLqV+PKbf15c7B7XhzwXZumbyY/CNFXpclIjVEARDiwsOM316aylPjerJ0ez6Xv/A9mTkFXpclIjVAASAAXJOWxHs/P4+SUsc1L87nkxW7vC5JRKqZAkD+V6/kpnx63yDOTWzK/dNW8IdPM3VcQCSIKQDkn8Q1juKdO/tz28AUpvywjetfWUjuoeNelyUi1UABIP+iXngYv7+iG8+O70VmziEufe47XS8gEoQUAFKhMb0S+eTe84nx31Z64jdZlJXpZnIiwUIBID+pU3xjZtw7iMvOTeCvszZw+xtLOKBTRUWCggJATqtRVATPju/F41d1Z8GW/Yx6dh4Lt6hLSKSuUwBIQMyMG/u35aO7BxIdGcENry7kma82UqouIZE6SwEgZ6RbQhNm3DeIK3sl8sxXm7j+1YXkHDzmdVkichYUAHLGGkVF8PR1vXhyXE/W7Cpg1LPfMXP1bq/LEpEzpACQszY2LYnPfzmYti0a8ot3lvHIh6s4cqLE67JEJEAKAKmUlNhoPvzFQO6+6BymZ+xk9HPfsWxHvtdliUgAFABSafXCw/jVyC5Mu3MAJaWOcS8t4OnZG3UbCZFaTgEgVaZ/+xbMfGAwY3om8NycTVw9aT6bcgu9LktEKqAAkCoVU78eT1/Xixdv7EN2/lEuff57Xvtui04XFamFFABSLUb1aM2XD17IBR3j+NM/1nHdyws0CL1ILRNQAJjZSDPbYGZZZvbIKd6PMrPp/vcXmVmKf3qkmU0xs9VmttLMLiq3zFz/Olf4Hy2rqE1SS8Q1juLVW9J4alxPNuYWMurZeUz+fqv2BkRqidMGgJmFAxOBUUAqcL2ZpZ402wQg3znXAfgf4An/9DsBnHM9gOHAU2ZW/jNvdM718j/2Vq4pUhuZGdekJTH7oQsZeE4sf/xsLde+vICsvYe9Lk0k5AWyB9APyHLObXHOFQHTgDEnzTMGeNP//ANgmJkZvsCYA+D/gj8IpFdF4VK3xMfUZ/Kt6Tx9bU825x1m9HPfMfGbLJ0pJOKhQAIgEdhZ7nW2f9op53HOlQAFQAtgJTDGzCLMrB2QBiSXW26Kv/vnd/7A+BdmdpeZZZhZRl5eXkCNktrJzLi6TxKzH7yQi7u25K+zNjDmhR9Yna0xiEW8EEgAnOqL+eRO3IrmeR1fYGQAzwDzgR8vFb3R3zU02P+4+VQf7px7xTmX7pxLj4uLC6Bcqe3iGkcx6cY0XrqpD/sOn2DMxO95/B9rOVqkq4hFalIgAZDNP/9qTwJyKprHzCKAJsAB51yJc+5Bfx//GKApsAnAObfL/7cQmIqvq0lCyMjurZn90IWM79eGV7/byvCn5/HNeh0KEqkpgQTAEqCjmbUzs0hgPDDjpHlmALf6n48FvnbOOTNraGbRAGY2HChxzq31dwnF+qfXAy4D1lRBe6SOadKgHn++qgfv/dt5NIgM5/Y3lnDP1GUah1ikBkScbgbnXImZ3QvMAsKB151zmWb2GJDhnJsBTAbeNrMs4AC+kABoCcwyszJgF//XzRPln17Pv86vgFersF1Sx/Rr15zPfzmYV+Zt5rmvs/h2Qx4Pj+jEzQPaEhGuy1VEqoM5V3fOyU5PT3cZGRlelyHVbNu+I/y/GZnM25hHt4QY/nRld3q3aeZ1WSJ1lpktdc79yxmY+mkltU5KbDRv3t6XiTf4DhJfNWk+j3y4SmMRi1QxBYDUSmbGpee2Zs7DF3HXBe35YGk2Q56cy9sLt+tKYpEqogCQWq1RVAS/Gd2Vz+8fTNfWjfndx2u4/PnvWbLtgNelidR5CgCpEzrFN+bdOwfwwg29yT9axLiXFvDLd5ezu0DjEYucLQWA1BlmxmXnJjDn4Qu5b2gHvsjcw9Anv+XZrzZxvLjU6/JE6hwFgNQ5DSMjeHhEZ+Y8dCFDusTxP19tZOiTc/lkxS7q0lltIl5TAEidldy8IZNuTOPdOwfQLDqS+6et4OoX52tMYpEAKQCkzjvvnBbMuHcQfxl7Ltn5x7h60nzumbqMnQeOel2aSK2mC8EkqBw5UcLL87bwyrzNlJXBrQPbcs+QDjRtGOl1aSKeqehCMAWABKU9Bcd56ssNfLAsm5j69bh3SAduPq8t9euFe12aSI3TlcASUlo1qc9fx/Xk818Opnebpjz++TqGPfUtHy7N1oVkIn4KAAlqXVvH8Mbt/Xjnjv40j47k4fdXMvrZ75izLldnDEnIUwBISDi/Qyyf3HM+L9zQmxMlpUx4M4NxLy3QFcUS0hQAEjLCwnwXks1+6EIev6o7Ow4cZdxLC7h9ymLW7NKwlBJ6dBBYQtaxolLemL+Nl77dTMGxYkb3aMVDwzvRoWVjr0sTqVI6C0ikAgXHipn83RYmf7+VY8WljOmVyP3DOpISG+11aSJVQgEgchr7D5/g5XlbeGvBNopLHdf0SeS+oR1Jbt7Q69JEKkUBIBKgvYXHmfTNZqYu3kFZmWNsWhL3DOmgIJA6SwEgcob2FBxn0twspi3eSZlzjEtP4u6LFARS9ygARM7S7oJjTPpmM9OX+ILgmj5J3D3kHNq20DECqRsUACKVtKfgOC996+saKi1zjOmZwN1DOtChZSOvSxP5SQoAkSqy99BxXpm3hXcW7eB4SSmju7fmniEdSE2I8bo0kVNSAIhUsf2HT/Da91t5e8F2Dp8oYViXltwztAN92jTzujSRf6IAEKkmBUeLeXPBNl7/YSsHjxYzoH1z7r6oA4M7xmJmXpcnogAQqW5HTpTw7uIdvPrdFnIPnaB7Ygy/uLADI7u3IjxMQSDeUQCI1JATJaV8tGwXL8/bwtZ9R0hp0ZA7L2jPNX2SNB6BeEIBIFLDSsscX2bu4cVvN7Mqu4DYRpHcNjCFmwa01QhlUqMUACIecc6xYMt+Xv52C99uzKNhZDjXpiczYVA7XVQmNUIBIFILrNt9iFfnbWHGyhzKnGN0j9bcObg9PZObel2aBDEFgEgtsrvgGFN+2Ma7i3ZQeKKEvinNuGNwey7uGq8DxlLlFAAitVDh8WKmL9nJlB+2sevgMdq2aMjPzm/H2LQkoqMivC5PgoQCQKQWKyktY1ZmLq99v4XlOw4SUz+C6/u14ZaBKSQ2beB1eVLHKQBE6oil2/OZ8sNWZq7ZA8Al3eK5/fx2pLdtpgvL5KxUFADaxxSpZdLaNiOtbTNyDh7jzQXbmLZ4J5+v3kO3hBhuG5jC5T0TdD2BVAntAYjUckeLSvh4eQ5vzN/KxtzDNI+O5Pp+ydw0oC2tm6h7SE6voj2AsAAXHmlmG8wsy8weOcX7UWY23f/+IjNL8U+PNLMpZrbazFaa2UXllknzT88ys+dM+7Yip9QwMoIb+rdh1gMXMPWO/qS1bcakuZsZ9MQ33P3OUhZu2U9d+iEntcdpu4DMLByYCAwHsoElZjbDObe23GwTgHznXAczGw88AVwH3AngnOthZi2BmWbW1zlXBrwI3AUsBD4HRgIzq65pIsHFzBjYIZaBHWLZeeAoby/czvQlvu6hzvGNufm8tlzVO1FnD0nAAtkD6AdkOee2OOeKgGnAmJPmGQO86X/+ATDM/4s+FZgD4JzbCxwE0s2sNRDjnFvgfD9d3gKurHRrREJEcvOG/GZ0Vxb+ehh/ueZcIsKN//x4Df3/PIdHP1lD1t5Cr0uUOiCQnwqJwM5yr7OB/hXN45wrMbMCoAWwEhhjZtOAZCDN/7fMv57y60w81Yeb2V349hRo06ZNAOWKhI4GkeFc2zeZcelJLNtxkLcXbOPdxTt5c8F2BrRvzk0D2jIitRWREQH19kqICSQATtU3f3KHY0XzvA50BTKA7cB8oCTAdfomOvcK8Ar4DgIHUK9IyDGz/z176HeXneC9jGzeWbSde6cuJ7ZRFOP7JjO+XzJJzXTvIfk/gQRANr5f7T9KAnIqmCfbzCKAJsABf/fOgz/OZGbzgU1Avn89P7VOETkLLRpF8YuLzuGuC9ozb2Me7yzawaS5WUycm8WQzi25oV8bhnRpqVtOSEABsAToaGbtgF3AeOCGk+aZAdwKLADGAl8755yZNcR3qukRMxsOlPx48NjMCs1sALAIuAV4vkpaJCIAhIcZQ7q0ZEiXluw6eIzpi3cwbclO7ngrg9ZN6nNd32SuTU8mQVcah6yArgMws9HAM0A48Lpz7nEzewzIcM7NMLP6wNtAb+AAMN45t8V/OugsfH3+u4AJzrnt/nWmA28ADfCd/XOfO00xug5ApHKKS8uYs24vUxfv4LtNeRhwUeeWXN+vDUM6xxERrmMFwUi3ghCRf7LzwFGmL9nJ9Iyd5BWeID4minFpyVzXN1njFAQZBYCInFJxaRlfr9/LtMU7+HZjHmUOBnWI5bq+yYzoFk9UhG47UdcpAETktHIOHuP9jGzey9jJroPHaNawHlf2TuS6vsl0aRXjdXlylhQAIhKw0jLHD1n7mJ6xk9mZuRSVlnFuUhOuTU/m8p4JNGlQz+sS5QwoAETkrOQfKeKj5bt4L2Mn6/cUEhURxsjurRiXlszAc1oQptNJaz0FgIhUinOOzJxDvJexk09W5FBwrJiEJvW5Ji2Ja/okkRIb7XWJUgEFgIhUmePFpcxem8v7S7P5blMezkG/lOaMTUtiVI9WNK6vLqLaRAEgItVid8ExPlq+iw+WZrMl7wj164UxslsrrklLYuA5sbriuBZQAIhItXLOsWLnQT5Yms2nK3M4dLyEVjH1ubJ3Ilf3SaRTfGOvSwxZCgARqTHHi0uZs24vf1+WzdyNeZSWObonxnBV7ySu6JlAXOMor0sMKQoAEfHEvsMnmLEih4+W72L1rgLCw4zBHWO5qnciI1Jb0SBSF5pVNwWAiHhuU24hf1++i0+W7yKn4DjRkeFc0r0VV/VO1PGCaqQAEJFao6zMsWjrAT5ans3M1XsoPFFCXOMoruiZwJW9EumeGIOGCa86CgARqZWOF5fy9fq9fLx8F99s2EtxqaN9XDRjeiZyRa8E2un6gkpTAIhIrVdwtJjP1+zm4+W7WLztAM5Bz6QmXNErkcvObU18TH2vS6yTFAAiUqfsLjjGZyt38/GKXWTmHMIMBrRrwRW9EhjVvRVNG0Z6XWKdoQAQkTprc95hZqzIYcbKHLbuO0JEmHFBpziu6JnAxanxNIoKZHDD0KUAEJE678f7EX26ModPV+aQU3CcqIgwhnZpyeU9ExjSuaVOKz0FBYCIBJWyMseyHfl8tmo3n63azb7DJ2gYGc6wrvFcdm5rLuwUR/16CgNQAIhIECstcyzasp9PV+3mizW7yT9aTKOoCIan+sJgUMfYkB7ZTAEgIiGhuLSMBZv389mqHL5Ys4dDx0toXN8XBpf2CM0wUACISMgpKinjh837+Meq3XyZWS4MusYzukdrBncKjTBQAIhISPsxDD5ftZsv1+ZScKyYxlERDOvaklE9gvuYgQJARMSvqKSM+Zv3MXP1Hmat3cPBo8U0jAxnaJeWjOremos6xxEdRKeWKgBERE6huLSMhVv2M3PNHmat2cP+I0VERYRxYac4RvVoxdAu8TRpULdHOFMAiIicRmmZY/HWA8zK3MMXa/aw59Bx6oUbA8+JZWT3VgxPjSe2Ud0by0ABICJyBsrKHCuyD/LFmj3MXLObnQeOEWaQntKckd1aMaJbPEnNGnpdZkAUACIiZ8k5x9rdh5iVmcuXmXtYv6cQgO6JMVyS2opLureiY8tGtfYW1goAEZEqsnXfEb7M3MMXmXtYvuMgAO1ioxmRGs+Ibq3ondyUsFo0uI0CQESkGuQeOs6Xa317Bgs276ekzBHbKIrhqfGMSI3nvHNaeH56qQJARKSaFRwrZu6GvXyZmcvcDXs5UlRKdGQ4F3aOY3hqPEM7x9OkYc2fUaQAEBGpQceLS1mwZT9fZuby1bpc8gpPEB5m9EtpzvDUeIanxpPcvGYOIisAREQ8UlbmWJl9kK/W5TJ7bS4bcw8D0KVVYy7uGs/FqfGcm9ik2o4bKABERGqJ7fuPMHutLwwytudTWuaIaxzFsC4tGdY1nkEdYqt0XAMFgIhILXTwaBFzN+Qxe10u8zbkUXiihKiIMM7vEMuwri0Z1iWeVk0qNxayAkBEpJYrKiljybYDzF6by5z1uew8cAyAbgkxvPWzfrQ4y6uQKwqAgO52ZGYjgWeBcOA159x/n/R+FPAWkAbsB65zzm0zs3rAa0Af/2e95Zz7L/8y24BCoBQoOVVxIiKhJNL/y//8DrE8enkqm/YeZs66vSzfkU/z6Mgq/7zTBoCZhQMTgeFANrDEzGY459aWm20CkO+c62Bm44EngOuAcUCUc66HmTUE1prZu865bf7lhjjn9lVhe0REgoKZ0Sm+MZ3iG1fbZ4QFME8/IMs5t8U5VwRMA8acNM8Y4E3/8w+AYea7JtoB0WYWATQAioBDVVK5iIhUSiABkAjsLPc62z/tlPM450qAAqAFvjA4AuwGdgBPOucO+JdxwJdmttTM7qrow83sLjPLMLOMvLy8AMoVEZFABBIApzox9eQjxxXN0w9fH38C0A542Mza+98/3znXBxgF3GNmF5zqw51zrzjn0p1z6XFxcQGUKyIigQgkALKB5HKvk4Cciubxd/c0AQ4ANwBfOOeKnXN7gR+AdADnXI7/717gI3xhISIiNSSQAFgCdDSzdmYWCVAPomUAAARkSURBVIwHZpw0zwzgVv/zscDXznd+6Q5gqPlEAwOA9WYWbWaNAfzTRwBrKt8cEREJ1GnPAnLOlZjZvcAsfKeBvu6cyzSzx4AM59wMYDLwtpll4fvlP96/+ERgCr4vdwOmOOdW+buBPvLfOzsCmOqc+6KK2yYiIj9BF4KJiAS5ii4EC6QLSEREglCd2gMwszxg+1kuHguE2kVnodhmCM12h2KbITTbfTZtbuuc+5fTKOtUAFSGmWWE2u0mQrHNEJrtDsU2Q2i2uyrbrC4gEZEQpQAQEQlRoRQAr3hdgAdCsc0Qmu0OxTZDaLa7ytocMscARETkn4XSHoCIiJSjABARCVFBHwBmNtLMNphZlpk94nU91cXMks3sGzNbZ2aZZna/f3pzM5ttZpv8f5t5XWtVM7NwM1tuZp/5X7czs0X+Nk/338MqqJhZUzP7wMzW+7f5ecG+rc3sQf//9hoze9fM6gfjtjaz181sr5mtKTftlNvWf5+15/zfb6vMrM+ZfFZQB0C50cxGAanA9WaW6m1V1aYEeNg51xXfTffu8bf1EWCOc64jMMf/OtjcD6wr9/oJ4H/8bc7HN2JdsHkW3512uwA98bU/aLe1mSUCvwTSnXPd8d2X7MfRB4NtW78BjDxpWkXbdhTQ0f+4C3jxTD4oqAOAwEYzCwrOud3OuWX+54X4vhAS+efR2t4ErvSmwuphZknApfjGnsY/Et1QfIMRQXC2OQa4AN9NGHHOFTnnDhLk2xrfjSMb+G853xDfQFNBt62dc/Pw3VSzvIq27Rh8Y60759xCoKmZtQ70s4I9AAIZzSzomFkK0BtYBMQ753aDLySAlt5VVi2eAX4FlPlftwAO+kemg+Dc5u2BPGCKv+vrNf9t1YN2WzvndgFP4rvF/G58ow4uJfi39Y8q2raV+o4L9gAIZDSzoGJmjYAPgQecc0E9/rKZXQbsdc4tLT/5FLMG2zaPAPoALzrneuMbdjVountOxd/nPQbfyIIJQDS+7o+TBdu2Pp1K/b8HewAEMppZ0DCzevi+/N9xzv3dPzn3x11C/9+9XtVXDc4HrjCzbfi694bi2yNo6u8mgODc5tlAtnNukf/1B/gCIZi39cXAVudcnnOuGPg7MJDg39Y/qmjbVuo7LtgDIJDRzIKCv+97MrDOOfd0ubfKj9Z2K/BJTddWXZxzv3bOJTnnUvBt26+dczcC3+AbmQ6CrM0Azrk9wE4z6+yfNAxYSxBva3xdPwPMrKH/f/3HNgf1ti6nom07A7jFfzbQAKDgx66igDjngvoBjAY2ApuB33pdTzW2cxC+Xb9VwAr/YzS+PvE5wCb/3+Ze11pN7b8I+Mz/vD2wGMgC3geivK6vGtrbC8jwb++PgWbBvq2BPwDr8Y0w+DYQFYzbGngX33GOYny/8CdUtG3xdQFN9H+/rcZ3llTAn6VbQYiIhKhg7wISEZEKKABEREKUAkBEJEQpAEREQpQCQEQkRCkARERClAJARCRE/X/uAmwfvdn2YgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xV9f3H8dcnOwRCAmElIWHvTSC4cIGiCKg/B07EQXF0Wdv6a3loq/XXVlurrcoQoUpFBEcFRS2OuhMSAoQNIZANScgie9zv749z0RACuUnuzU1uPs/HI4/ce873nPM5nPDOzRnfrxhjUEop5bm83F2AUkop19KgV0opD6dBr5RSHk6DXimlPJwGvVJKeTgfdxfQUFhYmBkwYIC7y1BKqQ5l27Zt+caYXo3Na3dBP2DAABITE91dhlJKdSgikna2eXrqRimlPJwGvVJKeTgNeqWU8nAa9Eop5eE06JVSysNp0CullIfToFdKKQ/X7u6jV0qpNnXiMCS/Ce2hy/bgcIhZ6PTVatArpTq3zb+Ew58C4u5KIDJGg14ppZwq74AV8pctgem/dHc1LqPn6JVSnVf8MvD2h8nO/xTdnmjQK6U6p/IC2LkOxt0IQWHurgabzVBWVeuSdWvQK6U6p6TXoKYcYu93dyWUVNZw32uJ3P96Ejab8y8Ka9ArpTqfulrY+jIMuAj6jnFrKSm5pVz74jd8cTCPmSN7Iy64JqwXY5VSncPxPXD4c+t14VEoyYSrn3FbOcYYPtp9jF+9lYy/rxev3xtL7KCeLtmWBr1SyvPZbLBhIeQf+GFa71Ew7Eq3lHPw+EmefH8vXx3KZ1xkd5bdPpnwkECXbU+DXinl+VI/s0J+7j9g1LXWNN8u4OXdpmWUV9fy9EcHWBOXRld/Hx6fM4rbp0Xj6+3as+ga9Eopzxe3DLr2gXHzwcfPLSUkpRfyi/U7OXqijNtjo3l45jBCg9qmFg16pZRnyzsIKVvgkt+4JeSNMfz90xSe//Qg/boHsvbeaZw32DXn4s9Gg14p5dm2LgdvP4i52y2bX/5lKn/75CDXTgjnyWvH0C3At81r0KBXSnmuikLYsRbG3ghde7X55jfuzOZPH+5nzvhwnr1pAl5e7ulPR++jV0p5rqQ19oeiFrf5prceKeCR9TuZMiCUZ24Y57aQB/1Er5TyVKceioq+EPqNa7PNVtfaeO27ozz3ySEiQwNZcUcMAb5te3dPQxr0SinPdOADKE6HWf/XZpv8fH8uT76/l9T8MqYP68Ufrx/bZnfWnIsGvVLKM8UthZAoGH61yzdVXFHD7zfu4Z3tWQzqFcTqu6Zw6YjeLt+uozTolVKeJ3s7pH8HVzzl8oeivknJ55cbdnL8ZBU/vXwoD146BD+f9nX506FqRGSWiBwQkRQRebSR+YtFZJeI7BCRr0VkVL15/2tf7oCIuOd5Y6VU5xK3DPy6wqQ7XLqZ95Ozuf2VeAL8vHnn/vP5+cxh7S7kwYFP9CLiDbwIzAQygQQR2WiM2Vuv2VpjzDJ7+7nAs8Ase+DPB0YD4cAnIjLMGFPn5P1QSinLyeOw+21rSL6A7i7bTOLRAh5ev5PJUaGsuSeWQD/3XnA9F0dO3UwFUowxqQAisg6YB3wf9MaYknrtg4BTHSrPA9YZY6qAIyKSYl/fd06oXSnliYyxhverKm3Z8oe2gK3GpbdUpuaVcu9riUSEBPLynTHtOuTBsaCPADLqvc8EYhs2EpEHgYcBP+CyesvGNVg2opFlFwGLAKKiohypWynlqQ5shnW3tm4dw2dDz8HOqaceYwyJaYU8smEnXiL8c+GUdnFXTVMcCfrG7vI/YwgUY8yLwIsiciuwBFjQjGVXACsAYmJinD+8ilKq44hbCt37w61v0niEOKDHQKeWVFNn49N9x1n+ZSrb04voEeTHygUxRPcMcup2XMWRoM8E+td7Hwlkn6P9OmBpC5dVSnVmOclw9CuY+QT0Ge3WUowxbD1SwMad2Xy4+xgFZdVE9ejCk/NGc8Pk/u3+dE19jgR9AjBURAYCWVgXV0/7u0pEhhpjDtnfzgZOvd4IrBWRZ7Euxg4FtjqjcKWUB4pfbvUTP+lOt5ZRU2fj0bd38XZSJoG+3swY1Yc54/px2Yje+Li473hXaDLojTG1IvIQ8DHgDawyxuwRkSeARGPMRuAhEZkB1ACFWKdtsLdbj3XhthZ4UO+4UUo1qjQPdq2HiXdAYKjbyqisqePB15P4dH8uP75sCPdfMpgufh37kSOHqjfGbAY2N5j2WL3XPz3Hsk8BT7W0QKVUJ7FtNdRVu6UDslPyS6t44F9JJKQV8OS1Y7hjWrTbanGmjv1rSinVcdnqrKdX62oAAwkrYcgM6DWszUtJO1HGyq+OsD4xA5sx/H3+ROaMD2/zOlxFg14p5R4JK+HDX50+7bxlbVrC8ZJKnvn4AO8kZeLj5cV1EyO4b/oghvTu2qZ1uJoGvVKq7dnqIO4liJhs9UcD4Bfk0u6Eq2ttZBaWA9Y93h/tPsaLn6dQW2e458KB3HfRIHoHB7hs++6kQa+UansHP4bCo3D54xB9nss3d+j4SRat2caR/LLTpl85ug+/uXpkh7kfvqU06JVSbS/uJQiOhJFzXb6pj/cc4+E3dxDo580frx9LF/v97/17dGFSlPvu7mlLGvRKqbZ1bLf1UNSM34O36yLoeEkly79IZdU3RxgX2Z3ld0ymX/dAl22vPdOgV0q1rfil4BPosoeiDueVsvyLw7y7PYs6m+GWqVE8PmeU24fzcycNeqVU2ynLh+QNMPE26NLDqasuLKvmuU8O8q/4dHy9hflTorj3ooEef/7dERr0Sqm2s2011FU55aGo4yWVHM0vI6e4ktS8Ul79Lo2TlTXcGhvFz2YMI6yrvxMK9gwa9EqptlFbDVtXwuDLodfw5i9eZ+OdpCy+SsknKa2QrKKK0+ZfOCSMJdeMZETfYGdV7DE06JVSbWPvv6H0GMx7odmLHs0v4+H1O0hKL6JvcACTB4Ryz4UDGdqnK/26BxIeEtDh+6NxJf2XUUq5njHWLZU9h1qf6B1ezPB6fDpPfbAPX2/h+fkTmDs+HJEW9lPfSWnQK6VcL2MrZG+Hq/8CXo5182uzGX6/aQ+vfpfGRUPDeOaG8fTt7plPrrqaBr1SyvXil1oDdY+/xaHm1bU2Htmwk407s7n3woH85uqReHnpp/iW0qBXSrVcTQUUZ527TUUB7N0I5z0A/k13FnaitIqH1+/ki4N5/HrWCBZfPEhP1bSSBr1SquXW3gxHvmi6nXjD1EXnbFJda+PVb4/y908PUVFTx5+uH8v8qVFOKrRz06BXSrVMzk4r5CcvhOgLzt02pD+EnD2092aX8NDaJFLzy7hkeC+WzB7lcV0Fu5MGvVKqZeKWgW8QzPgdBIa0eDVbjxRwz6sJBPn5sHrhFC4d3ttpJSqLBr1SqvlKc2H3WzBpQatC/pO9x3lwbRIRoYGsuSeWiJDO2emYq2nQK6WaL3GVfXzXHzVrsX05JayNTyerqILsogoO5ZYyOjyY1XdNoad2WeAyGvRKqeaprbKGARx6BYQNdWiROpth+ZeH+duWg/h6ezGgZxCRoYFcPLwXP75sKF39NYpcSf91lVI/qC6H6tJzt9m3CcryHOqYrKi8mqT0Ql78/DDb0gq5emxf/nDtWHoE+TmpYOUIDXqllKWiEJ6fAJVFTbcNGw6DL2t0ljGGdQkZrPr6CIdyrV8awQE+PHfzBOZN0O4L3EGDXillSXrNCvkZvwP/buduO+AiaCSwq2rrePy9PaxLyGBiVAiPXDGMydE9mNA/hEC/zjvwh7tp0CuloK4W4ldYAX7hz1u0ioyCcn6ybjvb04t46NIhPDxzmHZb0E5o0CulYP8mKMmEq59u1mJF5dVs2pnNpp05bD1aQBc/b5beNomrxvZzUaGqJTTolVLWw0+hA2DYLIcX2ZZWyOJ/bSPvZBVDe3flFzOHcd2kCCJDu7iuTtUiDgW9iMwCnge8gZXGmD81mP8wcC9QC+QBdxtj0uzz6oBd9qbpxpi5TqpdKeUMWUmQEQdX/hG8HDuP/sbWdB57bzfhIYH8+8ELmNC/5Q9NKddrMuhFxBt4EZgJZAIJIrLRGLO3XrPtQIwxplxE7geeBm62z6swxkxwct1KqfpqqwHTsmXjloJfN5h4e5NNU3JL+cdnh3hvRzbTh/XiH/Mn0r2Lb8u2q9qMI5/opwIpxphUABFZB8wDvg96Y8zn9drHAU3/xCilnCNxFbzfsguo34tdDAGNj7VaVlXLtrRC1sSlsWXvcfx9vPjxZUP42YxheOvF1g7BkaCPADLqvc8EYs/R/h7gw3rvA0QkEeu0zp+MMf9uuICILAIWAURFabekSjnMVgdf/w16j4ax/9OydXj5wMQ7TptUXF7Diq8O88XBPPblnKTOZgjp4stPLh/KgvOitbuCDsaRoG/sV3ajfyOKyO1ADHBxvclRxphsERkEfCYiu4wxh09bmTErgBUAMTExLfz7U6lO6MCHUJQON62BUa2//FVbZ2Pt1nSe3XKQkooapg7swQOXDGZSdCixA3voANwdlCNHLRPoX+99JJDdsJGIzAB+C1xsjKk6Nd0Yk23/nioi/wUmAocbLq+UaoG4pdA9CkbMbtVq0k+Usyk5m7e3ZZKaX8a0QT147JrRjApv/HSO6lgcCfoEYKiIDASygPnArfUbiMhEYDkwyxiTW296KFBujKkSkTDgAqwLtUqp1spJhrSvYeaTDt8tU9+x4kreT85mU3IOOzOsbg8mRYWw7PZJXDm6r3ZV4EGaDHpjTK2IPAR8jHV75SpjzB4ReQJINMZsBJ4BugIb7D8cp26jHAksFxEb4IV1jn5voxtSSjVPvH3gj0l3NN0WKKmsISmtkG1phcSlniAxrRBjYHR4MP971Qhmj+un98B7KIdOuBljNgObG0x7rN7rGWdZ7ltgbGsKVEo1ojQPdm2ASXdCYOhZmxWX1/DRnhw27czh28P52Ax4CYwKD+Znlw9jzvh+DOqlQ/Z5Or2yolRH9P3AH2d2FVxWVcsn+46zaWc2XxzMo6bOEN2zC4svHswFQ8KY0D+EIO3/vVPRo61UR1NbBYmvwJAZpw38cbKyhhc+T+HVb49SWWOjb3AAd543gLnjwxkX2V3PuXdiGvRKdTR73oXS4zDtJcAavWlDYgZ/+c8B8kuruW5iBLdMjSImOlR7j1SABr1SHYsxEPeSfeCPyykqr+bHb2znq0P5xESHsuquKYyL1H5n1Ok06JXqSNLjIGcnzH6W/cdPsui1bRwrruSp68Zw69QoPT2jGqVBr1QHkVFQjnz0LGE+wTx1dAxvb/qWrv4+vLFoGpOjz37njVIa9Eq1c7uziln+ZSrbk3fyhd8WXq67hg8PlDBtUE/+eP1Y+gQHuLtE1c5p0CvVTpVV1fLrt5N5PzmHrv4+vBy9Fa/jXtz146dYHBbt7vJUB6JBr1Q7dDS/jEVrEknJLeWnlw/lntjeBL90H4ycQ4CGvGomDXql2hGbzfDxnmP8+u1kvLyE1+6O5cKhYZCwEiqLYdoD7i5RdUAa9Eq1A1W1dby3I5uXv0zlUG4pI/p2Y8UdMUT17AI2G8Qvh/CJ0H+qu0tVHZAGvVJuZIzhg105/HHzfrKKKhjRtxvP3TyB2eP64evtZTU6/BnkH4TrXwa9fVK1gAa9Um6yO6uYJzbtZevRAkb2C+ap68Zw8bBeZ94LH78UuvaFUde6p1DV4WnQK9XGck9W8pePD7BhWyY9uvjxx+vHclNM/8bHX807ACmfwKVLwMev7YtVHkGDXqk2klVUwdvbMln+xWGq62zce+FAfnz5UIIDfE9vuOVx2P++9bqyGLz9IWZh2xesPIYGvVIuVF1rY8O2DN5NyiIxrRCAGSP78NvZIxkYFnTmAkUZ8O0/oN846DHImjbwYggKa8OqlafRoFfKBYwxfLovl6c27+NIfhnD+3TjkSuGMWd8ONE9Gwn4UxJetr7ftAZC+p+9nVLNoEGvlJMVlVfzk3U7+PJgHoN7BbF64RQuHd676QWry2DbP2HkNRryyqk06JVyopLKGu5ctZX9OSd57JpR3HFe9A+3STZl5zp9KEq5hAa9Uk5SVlXLwtUJ7M0uYdntk5kxqo/jC9ts1mDf/SZA/1jXFak6JQc/aiilzqW4ooZ7X01kR0YR/7hlYvNCHiDV/lDUtAf0oSjldPqJXnVuiash6bWm23XrBzeuBh//M2Z9k5LPIxt2knuyimdvGs9VY/v9MLO2CjYshJM5515/cSZ07QOjr2vmDijVNA161XlVl8Mnv4PAEOg59OztaivhwAfWWK3j538/uayqlmc+PsA/vz3KoF5BvHP/+Yzv32AYv91vW8sOuAh8ztFvfJeeMOFWfShKuYQGveq8ktdBZRHc8gZEn3/2dsbAi7HWWK3jbsZm4J3tWTz90X5yT1Zx1/kD+PWsEQT6eZ+5XNxL0GskLNikp2SU22jQq87JGIhbBv3GQ9R5524rQt3UH+G9+WE++OBdlh/tTXJmMeP7h7D09slnH8Yv7Vs4tgvmPK8hr9xKg151Toc/g/wDcO2yRkO4orqO+CMn2JZWSOLRQvZnhPG5VxASv5Ti4N/w7E3juXZCBF6N9U9zStxLEBgKY29y4Y4o1TSHgl5EZgHPA97ASmPMnxrMfxi4F6gF8oC7jTFp9nkLgCX2pn8wxrzqpNqVarn4ZRDUG8Zc//2k6lobXxzMY9PObD7Zd5zy6jq8vYRR/YKZN2UIeSdv4aqUVVx932AIiTz3+guPwoHNcMHPwK+La/dFqSY0GfQi4g28CMwEMoEEEdlojNlbr9l2IMYYUy4i9wNPAzeLSA/gcSAGMMA2+7KFzt4RpRyWfwgO/Qcu+V/w8aeksoY34tNZ9c0RjpdUEdrFl2snRnDVmL5MigolyN/+36T45/Dcati6Aq74w7m3sfVlQGDKvS7fHaWa4sgn+qlAijEmFUBE1gHzgO+D3hjzeb32ccDt9tdXAluMMQX2ZbcAs4A3Wl+6ard2vGFd6Dylfyxc+hvnrb+mEjY+BGV5LVu+OAu8/cgbfisrP9zH2rh0TlbVcv7gnvzfdWOZPqxX40+zdo+EUXMhYZV17v1cMhJg9LXQPaJlNSrlRI4EfQSQUe99JnCuR/fuAT48x7Jn/OSLyCJgEUBUVJQDJal27du/Q2ku9BwM5QWQ+mcYeyOEneMWxubYtcH6Cp8I3s2/HbHCtzufhd3Nz17YTZ3NcNXYfiyePpixkd2bXnj6L6E0D2oqzt0ufCJc9Eiza1PKFRwJ+sauNplGG4rcjnWa5uLmLGuMWQGsAIiJiWl03aqDqCyB3H3WaZFLfm2F4t9GW+fEZ/+19es3BuKWQp8xcN/nzbqbpai8muc+OcSauDR8vYVbp/bnngsHWeOyOqrPaFj4QQsKV8p9HAn6TKB+V3qRQHbDRiIyA/gtcLExpqrespc0WPa/LSlUdRDZSYCByBjrfdde1qf5HWvhsiXWXSitceRLyN0Dc19oVshvSMzgqc37KKmo4ZapUTw8cxg9u575lKtSnsiRvm4SgKEiMlBE/ID5wMb6DURkIrAcmGuMya0362PgChEJFZFQ4Ar7NOWpMhOs7xGTf5g2bTHUlEPSmtavP36Z9RTp2Bsdam6M4dktB/nlW8kM69OND35yEU9dN1ZDXnUqTQa9MaYWeAgroPcB640xe0TkCRGZa2/2DNAV2CAiO0Rko33ZAuBJrF8WCcATpy7MKg+VuQ3ChlndCpzSd6zVBcDWFVBX2/J1F6TCgQ8h5m7wPUd3AnZ1NsNj7+3h758e4sbJkay9N5aR/YJbvn2lOiiH7qM3xmwGNjeY9li91zPOsewqYFVLC1QdiDHWJ/phV545L3YxvHmb1e/LqHktW3/8CvDygZh7mmxaXFHDo28n8+HuY/xo+iAevWoEok+nqk5Kn4xVzlN4FMrzfzg/X9/wqyAkGj5eArveatn6Uz61encM7nfOZl8dyuOXG5LJK61iyeyR3HvRoJZtTykPoUGvnCdrm/U9opGg9/KGyx+Dr/5qPbDUEmFD4MKfn33zRRW8+HkKa+PTGdwriBV3ns+4yJCztleqs9CgV86TmQC+XaD3qMbnj73B+nKy/cdKWP5FKpt2ZmOAuy8YyK9mDSfA17vJZZXqDDTolfNkJkD4JPBumx+r3JOVPPPRAd5KyiTQ15sF5w/g7gsHEhES2CbbV6qj0KBXzlFTCTnJcN6DLttEUXk1GQUVZBdXsCermFe+PkJ1nY1FFw3i/ksGE9JFB+1QqjEa9Mo5ju0CW03jF2JbaUdGESu+PMxHu49hq/fc9MxRffjt1SMZEBbk9G0q5Uk06JVzfP+glPOCfv+xEh5/bw/xRwoIDvDhvumDmNg/lPCQACJCAvWhJ6UcpEGvnCMzAbr3b/LWR0fU2Qwrv0rlr/85SHCgD0tmj2T+1Ci6+uuPq1Itof9zlHNkJp7e7UEL5ZZU8tDa7Ww9WsCs0X156rox+sldqVbSoFetd/I4FKdD7I9atZr80ipuXRlPdlEFz940nusmRujTrEo5gQa9ar2sROt75JQWr6KovJrbV8aTWVjOqwunEjuop5OKU0pp0KvWy0wAL1/oN65FixeVV3PHK1tJzS/jlQUxGvJKOZkGvWq9zEToOwZ8m/+g0ql+aU6UVbHs9slcNLSXCwpUqnPToFetY6uDrCSYeNtZmxhjKK6oIbuokqLy6u+nf7TnGK99l8aQ3l15+c4Yx4byU0o1mwa9ap3cfVBTdsb5+aLyaj7afYxNydlsTy+ivLrujEVF4N4LB/LIldovjVKupEGvWufUg1L2J2Jr62w8+f5eXo9Pp9ZmGNCzCzfF9CcyNJDwkEB6BPl9P5BwWDd/Bvfq6p66lepENOhV62QmWkP7hQ6ksqaOn7yxnf/sPc4tU6O4LTaK0eHBeoukUm6mQa9aJysRImI4WVXLfa8lEpdawONzRrHwgoHurkwpZadBr1quogjy9lM8eC43L/uOlNxSnp8/gXkTItxdmVKqHg161XLZSQD8Kt6fbFPBqrumMH2Y3h6pVHujQa8cdrKyhpziSrKLKsguqqTvjs1cYoTjwaPYeOeF2l2wUu2UBr06p1NdBe/NKeFkZS1Xem1luGQCMNk7nuMBUfzrwSu0Z0ml2jH936ka1bCr4OsnRjAsoIhb4v6Bl6l3T/zUh0FDXql2Tf+HqtOUVNawZc9x/hWfxvb0Iq4c3Yf/u26s1VXwlsesRj9NtvqeB/Dycl+xSimHaNB3cjV1NvZml5CYVkhc6gm+OJhHda2NiJBA/nrjeK6fZO8quLoMtv0TRs6B0Gh3l62UagYN+k7svR1ZLHl3NyeragGICAnk1qlRzBkfzqSokNMfdNq5DiqLYdr9bqpWKdVSDgW9iMwCnge8gZXGmD81mD8deA4YB8w3xrxVb14dsMv+Nt0YM9cZhauWq62z8eeP9vPyV0eYMiCUu84fyOToUPp2D2h8AZsN4pdB+EToH9u2xSqlWq3JoBcRb+BFYCaQCSSIyEZjzN56zdKBu4BHGllFhTFmghNqVU5QXF7Dg2uT+DolnwXnRbPkmlH4ejdxnj31M8g/CNetsHoiU0p1KI58op8KpBhjUgFEZB0wD/g+6I0xR+3zbC6oUTnJ8ZJKFqzaSmpeGU/fMI6bYvr/MNMYSH7Tetq1oV3roWsfGH1d2xWrlHIaR4I+Asio9z4TaM7f7wEikgjUAn8yxvy7YQMRWQQsAoiKimrGqpWjjuaXcfsr8RSWVbN64RQuGBJ2eoODH8O75xjzdeYT4OPn2iKVUi7hSNA39re6acY2oowx2SIyCPhMRHYZYw6ftjJjVgArAGJiYpqzbtWEE6VVbN59jOc/OYjNwBuLpjEuMuTMhnEvQbdwWPwVSINTOeIFgY0so5TqEBwJ+kyg3t/4RALZjm7AGJNt/54qIv8FJgKHz7mQarbaOhsf7MohLvUExv6rMquogm8Pn6DOZhjRtxsv3DqJIb0b6f/9+F448gVc/jgEhZ05XynVoTkS9AnAUBEZCGQB84FbHVm5iIQC5caYKhEJAy4Anm5psepMpVW1bEjMYOVXR8gqqiCkiy/+PtYn8m4Bvvxo+iDmTghneJ9uZ+8XPn4p+ATC5LvarnClVJtpMuiNMbUi8hDwMdbtlauMMXtE5Akg0RizUUSmAO8CocAcEfm9MWY0MBJYbr9I64V1jn7vWTalHFRZU8fn+3PZlJzNp/tyqaq1MWVAKL+bO5rLR/TGy6sZd8aUnYDk9TB+PnTp4bqilVJu49B99MaYzcDmBtMeq/c6AeuUTsPlvgXGtrJGVU/uyUoWrEpgX04JYV39mD+lP9dOjGBiVGjLVrhtNdRWQuxi5xaqlGo39MnYDiTtRBl3vLKV/NIqXrptEleM6oNPU/fAn0tdDSSshEGXQu+RzitUKdWuaNB3ELuzirlrdQK1Nhuv3xvb8k/w9e19D07mwJznW78upVS7pUHfzhWVV/PcJ4dYE5dG727+rFt0HkN6d3POyuOWQo/BMGSmc9anlGqXNOjbqZziCt7bkc2yLw5TUlHD/KlR/GLmMKu7YGfISLAG9r7qGe1qWCkPp0HfjlTW1PHu9izeTcpi69ECAC4Y0pMls0cxsl+wczcWvxT8g2HCLc5dr1Kq3dGgbweKyqtZ810ar353lPzSaob27sovZg7jmvHhDHTFOKzFWdb5+djF4O+k00BKqXZLg74NGWNOe2gpo6CcV74+wvrEDMqr67h0eC8WTR/MtEE9zv5wkzMkrARjg6n3uW4bSql2Q4O+DaSfKGfl16m8tS0TPx8v+nUPpHugDwlHCxFg7oRwFk0fxIi+Tj4905jqcmukqOFXQ+gA129PKeV2GvQudCS/jL/+5wCbd+Xg7SVcMy6cIH9vcooqOX6ykrsvGMDCCwYSHhLYdkXtWg8VBTDtgbbbplLKrTTonWBnRhH/PZDH8L5dmRzdA39fL174LIXV3xzB19uL+6YPYuH5A88+glNbMQbilkHfsRB9vntrUUq1GQ36Fqqts/HVoXyWf3mYuNSC0+b5+82paG8AAAxhSURBVHhRXWfjhkmR/PLK4fQOdnPAn5L6X8jbB9cu1ZGilOpENOjPIae4gtXfHCU5s4i+wQGEhwTi7SUkpReyI72Isuo6+nUPYMnskVw/KZIj+WVsSysg7UQ586dEMTayu7t34XTxyyCoF4z5H3dXopRqQxr0dim5J9mRUQxYd8fEpRbw3o4sDDAmPJiEo4UcL8nBZgwj+wVzw+RIYgf1ZMbIPvjZuwXuEeTH5GgndE3gCicOw8GP4OJHwcdJD10ppTqETh/0tXU2ln+ZynOfHKSm7ofBrQJ9vbl9WjT3XDiQ/j26AFBnM9TU2Qjw9XZXuS0Xvwy8fCHmbndXopRqY5066I/kl/GL9TtISi9i9th+/HzmsO8H7QgN8qOr/+n/PN5egrdXBwz5ymLYsRbG3gDd+ri7GqVUG+uUQV9aVcsLn6Ww6usjBPh68fz8CcwdH+7ah5Tcafu/oLpU+5xXqpPy+KCvsxnyTlaRXVxBdlEFaSfKWf3NUfJLq7h+UgS/njWCPu3lrhhXsNVZp22izofwCe6uRinlBh4Z9GVVtfzqrWR2ZBRxvKSSWps5bf7k6FBWLohhQv8QN1XYhg58CEXpcMUf3F2JUspNPC7oa+tsPLQ2iS8O5jFnfDiRoYH06x5IREgg/UIC7N0P+Lq7zLYTtxS6R8Hw2e6uRCnlJh4V9MYYHt+4h88P5PHUdWO4LTba3SW5V04ypH0NM58Eb4861EqpZvCoESeWf5nK6/HpLL54sIY8WOfmfYNg0h3urkQp5UYeE/QpuaX8+aP9XDOuH7+6cri7y3G/0jzYtcEaWCSwnT7EpZRqEx7z9/yQ3l1ZddcUzhvUEy8vD71NsjkSV0Fdtd5SqZTynKAHuHR4b3eX0D7UVkHiK9ag32FD3V2NUsrNPObUjapnz7tQehym6ad5pZQGvecxBuJegrDhMPhyd1ejlGoHHAp6EZklIgdEJEVEHm1k/nQRSRKRWhG5ocG8BSJyyP61wFmFq7NIj4OcnRD7I+1zXikFOBD0IuINvAhcBYwCbhGRUQ2apQN3AWsbLNsDeByIBaYCj4uI3gLiSvFLISAExs93dyVKqXbCkU/0U4EUY0yqMaYaWAfMq9/AGHPUGJMM2BoseyWwxRhTYIwpBLYAs5xQt2pMUTrs2wSTF4BfkLurUUq1E44EfQSQUe99pn2aI1qzrGqurS8DAlPuc3clSql2xJGgb+xEr2lkWouXFZFFIpIoIol5eXkOrlqdproMkl6FkXMgpL+7q1FKtSOOBH0mUD85IoFsB9fv0LLGmBXGmBhjTEyvXr0cXLU6zc43rAFGpj3g7kqUUu2MI0GfAAwVkYEi4gfMBzY6uP6PgStEJNR+EfYK+zTlTDYbxC2D8InQf6q7q1FKtTNNBr0xphZ4CCug9wHrjTF7ROQJEZkLICJTRCQTuBFYLiJ77MsWAE9i/bJIAJ6wT1POdPgzOHHI+jSvt1QqpRoQYxw93d42YmJiTGJiorvL6FjWXA/H98DPdoGPn7urUUq5gYhsM8bENDbPo/q66TRsNqgssl4XHoHDn8KlSzTklVKN0qDviN57wLr4eoq3P8QsdF89Sql2TYO+oynOhOT1MOIaGDjdmtZrBASFubcupVS7pUHf0Wx9GTAw648QEuXuapRSHYD2XtmRVJfBtn/aH4rSkFdKOUaDviNJftO6CBt7v7srUUp1IBr0HYUx1kNR/SZA1DR3V6OU6kA06NszY8BWZ32lfAr5B2Da/fpQlFKqWfRibHu25lpI/e8P77v2gdHXua0cpVTHpEHfXmUkWCE/5gboNdyaNuAi8PF3a1lKqY5Hg769insJ/LvDnOfBv6u7q1FKdWB6jr49Ks6Cve/BpDs05JVSraZB3x4l2B+KmrrI3ZUopTyABn17U11uPRQ1YjaERru7GqWUB9Cgb2+S34SKQn0oSinlNHoxtqX2b4Z3F4Ot1rnrra2EvuMg+nznrlcp1Wlp0LeEMfDl0xAQDKPmOX/9Y/5HH4pSSjmNBn1LZGyF7O0w+68w5V53V6OUUuek5+hbIn4pBHSH8be4uxKllGqSBn1zFWfC3o0waQH4Bbm7GqWUapIGfXOdGvhj6n3urkQppRyiQd8cp+5x14E/lFIdiF6Mbcw3z0Pc0jOn11XrwB9KqQ5Hg76hqpPw5V8gJBoiJp45PyRaB/5QSnUoGvQN7XgDqkpgznMQGePuapRSqtX0HH19NhvEL4PIKRrySimPoUFfX8oWKDgMsYvdXYlSSjmNQ0EvIrNE5ICIpIjIo43M9xeRN+3z40VkgH36ABGpEJEd9q9lzi3fyeJegm7hrunWQCml3KTJc/Qi4g28CMwEMoEEEdlojNlbr9k9QKExZoiIzAf+DNxsn3fYGDPByXU7X+4+a+i+yx8Db193V6OUUk7jyMXYqUCKMSYVQETWAfOA+kE/D/id/fVbwAsibdwrV3kBrL6q5ctXFIJPAExe6LyalFKqHXAk6COAjHrvM4HYs7UxxtSKSDHQ0z5voIhsB0qAJcaYrxpuQEQWAYsAoqJa+CCSl/cPg2i31ODLoUuP1q1DKaXaGUeCvrFP5sbBNjlAlDHmhIhMBv4tIqONMSWnNTRmBbACICYmpuG6HRPQHW56rUWLKqWUJ3PkYmwm0L/e+0gg+2xtRMQH6A4UGGOqjDEnAIwx24DDwLDWFq2UUspxjgR9AjBURAaKiB8wH9jYoM1GYIH99Q3AZ8YYIyK97BdzEZFBwFAg1TmlK6WUckSTp27s59wfAj4GvIFVxpg9IvIEkGiM2Qi8AqwRkRSgAOuXAcB04AkRqQXqgMXGmAJX7IhSSqnGiTEtOyXuKjExMSYxMdHdZSilVIciItuMMY0+0q9PxiqllIfToFdKKQ+nQa+UUh5Og14ppTxcu7sYKyJ5QForVhEG5DupnI6iM+4zdM797oz7DJ1zv5u7z9HGmF6NzWh3Qd9aIpJ4tivPnqoz7jN0zv3ujPsMnXO/nbnPeupGKaU8nAa9Ukp5OE8M+hXuLsANOuM+Q+fc7864z9A599tp++xx5+iVUkqdzhM/0SullKpHg14ppTycxwR9UwOYewoR6S8in4vIPhHZIyI/tU/vISJbROSQ/Xuou2t1NhHxFpHtIvK+/f1A+2D0h+yD0/u5u0ZnE5EQEXlLRPbbj/l5nn6sReTn9p/t3SLyhogEeOKxFpFVIpIrIrvrTWv02Irl7/Z8SxaRSc3ZlkcEfb0BzK8CRgG3iMgo91blMrXAL4wxI4FpwIP2fX0U+NQYMxT41P7e0/wU2Ffv/Z+Bv9n3uRBrkHpP8zzwkTFmBDAea/899liLSATwEyDGGDMGq2v0+Xjmsf4nMKvBtLMd26uwxvMYijXs6tLmbMgjgp56A5gbY6qBUwOYexxjTI4xJsn++iTWf/wIrP191d7sVeBa91ToGiISCcwGVtrfC3AZ1mD04Jn7HIw1psMrAMaYamNMER5+rLHGyQi0j1bXBWtIUo871saYL7HG76jvbMd2HvCascQBISLSz9FteUrQNzaAeYSbamkzIjIAmAjEA32MMTlg/TIAeruvMpd4DvgVYLO/7wkUGWNq7e898ZgPAvKA1fZTVitFJAgPPtbGmCzgL0A6VsAXA9vw/GN9ytmObasyzlOC3pEBzD2KiHQF3gZ+1nCwdU8jItcAufZxh7+f3EhTTzvmPsAkYKkxZiJQhgedpmmM/Zz0PGAgEA4EYZ22aMjTjnVTWvXz7ilB78gA5h5DRHyxQv51Y8w79snHT/0pZ/+e6676XOACYK6IHMU6LXcZ1if8EPuf9+CZxzwTyDTGxNvfv4UV/J58rGcAR4wxecaYGuAd4Hw8/1ifcrZj26qM85Sgd2QAc49gPzf9CrDPGPNsvVn1B2hfALzX1rW5ijHmf40xkcaYAVjH9jNjzG3A51iD0YOH7TOAMeYYkCEiw+2TLgf24sHHGuuUzTQR6WL/WT+1zx59rOs527HdCNxpv/tmGlB86hSPQ4wxHvEFXA0cBA4Dv3V3PS7czwux/mRLBnbYv67GOmf9KXDI/r2Hu2t10f5fArxvfz0I2AqkABsAf3fX54L9nQAk2o/3v4FQTz/WwO+B/cBuYA3g74nHGngD6zpEDdYn9nvOdmyxTt28aM+3XVh3JTm8Le0CQSmlPJynnLpRSil1Fhr0Sinl4TTolVLKw2nQK6WUh9OgV0opD6dBr5RSHk6DXimlPNz/A8pjHkL5ZjW7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores)\n",
    "plt.plot(test_scores)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MSE VS Cross Entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xW9d3/8deHhL333gQFQUADCC7c1AFOVlVQFOqutrVqe9ufettWa2vVohYBRRRBHIAW5XbXhSQM2UhARlgJK0ASMr+/P86lxpBxBa7kXOP9fDzyyHXOda5zfQ4neXNyru8w5xwiIhL5qvldgIiIhIYCXUQkSijQRUSihAJdRCRKKNBFRKJEvF9v3KxZM9epUye/3l5EJCItWbJkj3OueUnP+RbonTp1Ijk52a+3FxGJSGa2pbTndMtFRCRKBBXoZjbUzNabWYqZ3VfKNiPMbI2ZrTazmaEtU0REylPuLRcziwMmARcAqUCSmc13zq0psk0CcD9wunNuv5m1qKyCRUSkZMFcoQ8AUpxzm5xzucAsYHixbW4GJjnn9gM459JCW6aIiJQnmEBvC2wrspwaWFdUd6C7mX1pZovMbGioChQRkeAE08rFSlhXfESveCABGAK0Az43s17OuQM/25HZBGACQIcOHSpcrIiIlC6YK/RUoH2R5XbAjhK2meecy3POfQ+sxwv4n3HOTXbOJTrnEps3L7EZpYiIHKNgrtCTgAQz6wxsB0YBY4ptMxcYDbxkZs3wbsFsCmWhIiK+2bsRVsyG8oYbr9MEEm+E+Jo/rcvPheRpkLX3p3UnDIW2p4a8zHID3TmXb2a3AwuBOGCac261mT0MJDvn5geeu9DM1gAFwO+cc3tL36uISAR5715I+ZCS70AX5WDnt3D5c2Dm/Qfw7t2w/JWfv7Z+K38CHcA5twBYUGzdg0UeO+CewJeISPTYu9EL8yH3w5ASu+H85NPH4NM/Q7PucOY98NXTXpifdS+c+4dKL9W3rv8iIhEhaSpUi4dTx5W/7dn3wp7v4KOH4HAafPM89Lzc+8+gCijQRURKk5vpXWH3GObdJimPGQz/F+zfDN88B236ebdfqlXNKCsKdBGR0qycA0cyYMDNwb+mem0YNRMWTYLTboUadSqvvmIU6CIiJXEOFk+Blr2gw6CKvbZ+S7jg4cqpqwwKdBGRH+TnePfAAdLXw+6VcNlT3q2UCKBAFxEByDkE037hhfgPajWE3tf4V1MFKdBFRAoL4M2bIG0NXPzETx+ANk2AGnX9ra0CFOgiIh88CN+974V5RT4ADTMKdBGJHoWFsGQaZGwP/jVZe2Dpy9D/5ogOc1Cgi0g0+fgR+OIfXkegcrvpF9FzOAz9a6WVVVUU6CISHZa/5oX5qePg0n9GTMuUUNIk0SIS+bZ8De/cCZ3P8u6Dx2CYg67QRSScbfwY3r8f8o+Uvd3hNGjYHq6ZDnHVq6a2MKRAF5HwlLYWZl8P9VpA+4FlbxtfE8642xuPPIYp0EUk/GTugZkjvXFQxs6Hhu38rigiKNBFpHLlZnrd6IPmYOEf4PBuGLdAYV4BCnQRqTxZ+2DK+bBvY8Vfe/WL0C70s/pEMwW6iFSO/Fx4/XrI2AbDJ0GdZsG/tkEbaH1y5dUWpRToIhJ6zsGC38Lmz+GKydBnpN8VxQQFukgkSvkQNn3qdxWlO7QbVr4OZ/5GYV6FFOgikWbjJ/DqCKgWF+jiHqb6Xgvn/NHvKmJKGP80iMhR9myAOWOh+YkwfiHUrO93RRJG1PVfJFJk7YOZIyCuBoyZpTCXo+gKXSRSvD3RGxZ23LvQqIPf1UgY0hW6SCTYvRo2/B+ccz+0H+B3NRKmFOgikWDxCxBfC04Z63clEsYU6CLhLvsArJgNva+O+cGnpGwKdJFw9+1rkJflTZEmUoagAt3MhprZejNLMbP7Snh+nJmlm9nywNdNoS9VJAYVFnq3W9oNgDZ9/a5Gwly5rVzMLA6YBFwApAJJZjbfObem2KaznXO3V0KNIrFr0yfewFZD7ve7EokAwVyhDwBSnHObnHO5wCxgeOWWJSKAd3Vetzn0HOZ3JRIBggn0tsC2IsupgXXFXWVmK8zsDTNrX9KOzGyCmSWbWXJ6evoxlCsSQ/Zvge/e91q2xNf0uxqJAMEEekmzrbpiy+8AnZxzJwMfAtNL2pFzbrJzLtE5l9i8efOKVSoSa5KnglWDxBv8rkQiRDCBngoUveJuB+wouoFzbq9zLiew+AKgUelFjkdeNiydASderBl7JGjBBHoSkGBmnc2sBjAKmF90AzNrXWRxGLA2dCWKxKBVb0H2Phgwwe9KJIKU28rFOZdvZrcDC4E4YJpzbrWZPQwkO+fmA3ea2TAgH9gHjKvEmkWim3OweLI3omKnM/2uRiJIUINzOecWAAuKrXuwyOP7AbWrEgmF7Utg53K4+Amwkj7CEimZRlsUqYi8I7BjGUe3CwihrydBjfrQZ1TlvYdEJQW6SLDysuGlS2F7cuW/14CJGu9cKkyBLhIM52De7V6YX/wENEuovPeyatA2sfL2L1FLgS4SjM8eh1VvwHl/ggEaJEvCkwJdpCTfLYQNH3iP87Jg+avQZzSccbe/dYmUQYEuUlzeEZh7C+RmQfXa3roTLoHLnlKrEwlrCnSR4tbMhay9cN3b0PVcv6sRCZomuBApbvFkaJoAnYf4XYlIhSjQRYravsT7GnAzVNOvh0QW/cSKFLV4ClSvq049EpEU6CI/yNwLq970wrxWQ7+rEakwBbrID5a9DAU5amcuEUuBLgKw73v48mlvdMMWPfyuRuSYKNBFjmTAzJHgCr225iIRSu3QJbYV5MOcG2DfRrhuLjTt6ndFIsdMgS6x5+tnYc967/H+LbDpE7jsaeisySQksinQJbbsXg0L74dajSC+JmAw5AE4dazflYkcNwW6xJbFL0B8LbhzGdRp4nc1IiGlD0UldmQfgBWzodfVCnOJSgp0iR3fvuYNhat25hKlFOgSGwoLIWkKtOsPbfr6XY1IpVCgS2zY9AnsTYEBE/yuRKTSKNAlNix+Aeo0g57D/a5EpNIo0CX6rX8PvnsfTh0XaKooEp0U6BLddq2CN8Z7983P/I3f1YhUKgW6RK/DafBaYCjcUa9BjTp+VyRSqdSxSKJT3hGYNcabG/SG96BBa78rEql0CnSJPs7BvNsgNQlGzFAzRYkZQd1yMbOhZrbezFLM7L4ytrvazJyZJYauRJEK+u/fYNUbcN6D0HOY39WIVJlyA93M4oBJwC+AnsBoM+tZwnb1gTuBb0JdpEjQVr0FnzwKfUbDGff4XY1IlQrmlssAIMU5twnAzGYBw4E1xbZ7BHgc+G1IKxQpy6bP4K0JkHPIW87LgvaneRNVmPlbm0gVCybQ2wLbiiynAgOLbmBm/YD2zrl3zazUQDezCcAEgA4dOlS8WpGi9myA16+Dui2g99Xeuhp1YcBEtTeXmBRMoJd0meN+fNKsGvAkMK68HTnnJgOTARITE105m4uULmufN21ctepw7ZvQuKPfFYn4LphATwXaF1luB+woslwf6AV8at6fuK2A+WY2zDmXHKpCJcYV5MHmz73vAF89AxnbYOw7CnORgGACPQlIMLPOwHZgFDDmhyedcxlAsx+WzexT4LcKcwmZwkJ440ZYO//n66/4N3Q4zZ+aRMJQuYHunMs3s9uBhUAcMM05t9rMHgaSnXPzy96DyHH65FEvzIc8AAnne+tqN4Emnf2tSyTMBNWxyDm3AFhQbN2DpWw75PjLEgn4djZ8/gSccj2cfa9aroiUQWO5SPjavgTm3w6dzoSL/64wFymHAl3C16d/hVqNYMTLEF/D72pEwp4CXcLTvk2w4QNIvFETOosESYEu4SlpKlSL8yalEJGgKNAl/ORmwbIZ0OMyDXsrUgEKdAk/K+fAkQxN6CxSQQp0CS/OQdIL0OIk6DDI72pEIoomuBB/HNwB25eWsH477FoJlz6pZooiFaRAl6rnnDc93I5lJT9fuzH0HlG1NYlEAQW6VL3tS7wwH/IAnPCLo5+v1xJq1qv6ukQinAJdqt7iyVCjPgy6FWrW97sakaihD0Wlah1Oh9VvQ98xCnOREFOgS9VaOh0KcqH/TX5XIhJ1FOhSdQryIflF6Hw2NO/udzUiUUeBLlXnu/fgYKo6DIlUEn0oKpUn5xC8chVsWxxY4aBBO+g+1NeyRKKVAl0qR2EBvDEeUpNh0G1QvY63vuu5EKcfO5HKoN8sqRwfPAgbFsLFT8CAm/2uRiQmKNAl9JZMh6//5d0rV5hLmFi/6xApaYf9LgOAXm0b0LFp3ZDvV4EuofX9f+E/90DX8+Civ/hdjQgAi7/fxy+nLCKvwPldCgD/e3kvBbqEub0bYfZ10LQbXPOi7pVLWNiyN5OJM5Jp37gOT4/uR414/xv3tahfs1L2q984CY3s/TBzhDfL0OhZUKuh3xWJcPBIHuOnJ1PoYOq4/nRuFvqr4nCiQJfjV5AHc8bB/i0wdj406ex3RRKjjuQV8Pj769mZkQ3A93sy2bwnkxnjB0Z9mIMCXY6Xc/DevbDpUxj+LHQc7HdFEqOcc/z+zRXMW76DhBb1MINqZvx9RB8GdW3qd3lVQoEux2fxZEieBqf/Gvr90u9qJIb96+MU5i3fwe8uOoHbzunmdzm+UKBLxTjnfQFs/Bjevw9OvBTO+5O/dUlM+8+Knfz9g++4sl9bbh3S1e9yfKNAl+Ad3AnPDvQmcP5Bq95wxb+hmv8tByQ2rUg9wG/mLOfUjo35y1W9sRieujCoQDezocBTQBwwxTn312LP/wq4DSgADgMTnHNrQlyr+O27970wH3yHN0FFXHXod61mFxLf7MzI5qbpyTStW5N/X3cqNePj/C7JV+UGupnFAZOAC4BUIMnM5hcL7JnOuecD2w8D/gFoBKZos/Ejb3CtCx7RBM7iu6zcfG6ankxWbgFv3jKQZvUqp213JAnmCn0AkOKc2wRgZrOA4cCPge6cO1hk+7pAeHTHktApyINNn8FJlyvMJST2ZebyRcoenDu2uJi/fAdrdx5k6tj+nNBKs19BcIHeFthWZDkVGFh8IzO7DbgHqAGcW9KOzGwCMAGgQ4cOFa1V/JSaBDkHodv5flciUSAjO49rnv+KjemZx7WfBy/tyTkntghRVZEvmEAv6XLsqP9SnXOTgElmNgb4IzC2hG0mA5MBEhMTdRUfSVI+AovzZhsSOQ75BYXcPnMpW/Zm8e/rTqVbi2P7DKZOjThaN6wd4uoiWzCBngq0L7LcDthRxvazgOeOpygJQykfQrv+ULuR35VIhHvk3TV8vmEPj13Vm4tOauV3OVElmEBPAhLMrDOwHRgFjCm6gZklOOc2BBYvATYg0eNwOuxcDuf80e9KJAK9nrSND9buBryu+Z9v2MOEs7owsr9uu4ZauYHunMs3s9uBhXjNFqc551ab2cNAsnNuPnC7mZ0P5AH7KeF2i0SwTZ9437ud528dEnHeXbGDe99cQfsmtalXszoA153Wkd8PPdHnyqJTUO3QnXMLgAXF1j1Y5PFdIa5LwknKh1CnKbTu63clEkGWbzvAb17/lsSOjXn15oEx30a8Kqh7n5StsND7QLTrueoNKkHbcSCbm19Opnl9dfipSur6L2VLWw1Ze7wZiESCkJnjdfjJzi3g1ZsG0lQdfqqMLrmkbLtWet/bnupvHRIRCgsdd89ezrpdB3lmTD+6t1SHn6qkQJeypa2FuBrQpIvflUgEeHzhev5vzW7+59KenHOCOvxUNd1ykbKlr4OmCZofNIY55/giZQ9pB3PK3G7L3kye/2wjYwZ2YNzgTlVTnPyMfkulbGnroH1/v6sQH7301WYeeie4wVPPTGjGQ8NOiukhbP2kQJfS5RyGjK1wyvV+VyI++WR9Go+8u4YLe7bkj5f0LHNbM2jbqDbVqinM/aJAl9Klr/e+t1AnkFi0ftch7pi5jBNbNeDJkX2pW1NxEe50hqR06Wu97817+FuHhNSq7Rk88/EGcvMLy9xu9Y6D1K4Rx9RxiQrzCKGzJKVLWwtxNaFJZ78rkRDZcSCbcS8mUVBYSPsmdcrctlOzuvzxkh4a0TCCKNCldOnroFl3qKZeftEgMyef8dOTyckr4K1bB5OgNuJRR4EupUtbBx0H+V2FBBQWOrLzCo7ptQ64e/Zy1u86yLRx/RXmUUqBLiU7chAOpkJzfSAaDjKy8hj1wiLW7jxY/sZl+NNlPRmiDj9RS4EuJfuxhYs+EPVbXkEht81cSkraIe4+vzu1axxbB+/2jeswtJcmlIhmCnQp2Y8tXHSF7ifnHA+9s5ovUvbwt6tP5prE9uW/SGKWxnKRkqWtg/ha0LiT35XEtOlfbeaVRVuZeHYXhbmUS4EuJUtfqxYuPvt0fRoPv7uG83u05N6L9JeSlE+BLiVLW6f75z7asNvrpXlCqwY8NaovcepOL0FQoMvRsg/AoR26f+6TvYdzuHF6EjWrxzFlrHppSvD0kyKend/Cp3+F/COQm+mt0xV6lcvJL+CWV5ay+2AOsyecRttG6qUpwVOgC2Rsh1dHQEEuNO3qret6HnRQp6Kq5JzjgbdWsXjzPp4e3Y9+HRr7XZJEGAV6rMvNhNdGed/H/x+0LHuIVKk8z3+2iTeXpnLXeQkM69PG73IkAinQY8WRDHCu2EoH8++A3atg9GyFeRU7kldATmDEwy827OHxheu49OTW/Pr8BJ8rk0ilQI92hYXw9gRYOaf0bS76C3S/sOpqEhZt2sv4l5LIzP1pbJY+7RvxxDV9NNuPHDMFerT79M9emCfe6M0NWlzDdtDjsqqvK4Zt2ZvJr15ZQsuGtfjlwI4A1IgzLuvThlrV1e5fjp0CPZqteB3++zfodx1c8g9vjjDxVUZ2Hje+lATAtLH96dSsrs8VSTRRoEeTtHWw9SvvcW4mfPQIdDxDYY7X63L7gexytzu9a7NjCtmM7DwWrtpFXmHZswC9++1Otu7LYsb4gQpzCTkFerRIWwtTLoDcQz+ta9YdRs6A+Br+1RUGZn6zlQfeXhnUtk3q1mDurafToWnZs/kUdSSvgBteXMzSrQfK3TaumvGXK3tzWpemQe9fJFhBBbqZDQWeAuKAKc65vxZ7/h7gJiAfSAdudM5tCXGtUprMPTBzJNSo4zU9rNPEW1+nKcRV97c2n32VsocH561iyAnNeeyqkynr75TdB3O4duo3jJ+exJu3DqZBrfL/7Zxz3PfmCpZuPcCTI/twetdmZW5fs3ocDWvH9jmRylNuoJtZHDAJuABIBZLMbL5zbk2RzZYBic65LDO7BXgcGFkZBUsx+Tkw65dweDfcsEBND4vYlH6YX72yhC7N6/LM6H7ULyegWzSoxXPXnsL1Uxdzx8xlTB2bSHxc2aNjTPokhbnLd/DbC7tzRb92oSxfpMKCuUIfAKQ45zYBmNksYDjwY6A75z4psv0i4NpQFimlcA7euQu2LYKrX4S2p1bZW+fmF/K7N74l7WAO4N1KmHBWF87q3jxk7/HFhj38+78byS8o3n4+OBvTDxMfV42pY/uXG+Y/GNy1GY9c3ov731rJsH99WebVdKFzfPP9Pq7o15bbzul2TDWKhFIwg3O1BbYVWU4NrCvNeOC9kp4wswlmlmxmyenp6cFXKSX74kn49jUY8gD0urJK3/q9VTuZt3wHWbn5FBQ6vt+TycQZS1i9IyMk+1+78yATZySzMe0wBYXumL66t6zP1LGJ5c5uX9zoAR34w8U9qFczvsz9OwcjE9vzlyt7q+24hIVgrtBL+kkt8ZLJzK4FEoGzS3reOTcZmAyQmJh4bJdd4ln7Dnz0EPS6Gs6+t8rf/uWvt9CpaR3evvV0qlUz0g4eYfikL7lpejLzbjudFg1qHfO+0w/lcNP0ZOrXqs7bt51Oy+PY17G6+awu3HxWlyp/X5HjEUygpwJFp0ppB+wovpGZnQ/8ATjbOZcTmvKkRDu/hbcmQNtEGP6vKm+SuGp7Bku27Od/Lu1JtcA43S0a1GLK2ESuef5rbn45mcnXJxJ/DGN4FxQ6Jr6yhH2Zucz51SBfwlwkUgUT6ElAgpl1BrYDo4AxRTcws37Av4Ghzrm0kFcpPzm4E2aOgtpNYNRMqF71w6vO+HoLtavHcfWpP/8Q8KQ2DfnnyL5MfGUJA//80XG9x/PXnkKvtg2Pax8isabcQHfO5ZvZ7cBCvGaL05xzq83sYSDZOTcf+BtQD5gTuJe41Tk3rBLrjk25WTBrtDfQ1viFUL9llZdwICuXucu3c+Up7Ur8wPDCk1ox86bT2JB2qIRXB6dbi3oMLqf5n4gcLah26M65BcCCYuseLPL4/BDXJcUVFsLcW2DHcu/KvFVvX8qYk5xKTn4h1w/qWOo2g7o2ZVBXdZwRqWrqKRopPvsrrJkLFzwMJ17sSwmFhY4Zi7YwoFMTerRu4EsNIlI6zSkaCVbMgc8eg77XwuA7fSvjs+/S2bovi+vKuDoXEf8o0MPdtsUw7zboeDpc+qSvg2xN/3ozLerXZGivVr7VICKlU6CHswNbYdYYaNAaRvg7yNbmPZl89l06YwZ2oHo53eFFxB+6h+63jFRY8DvI3n/0c/s3Q34ujPsP1PX3Q8ZXFm0hzowxAzr4WoeIlE6B7qecw94Ezfu+h7anHP18i55w5j3Q/ISqr62I7NwCXk/exkW9Wh1XD1ARqVwKdL8UFnq9PXevhjFzICF8W37OW76dg0fyGTuok9+liEgZFOhVKXMvFOR6jxdNgvX/gaGPhWWYH8jK/XFG+pe/3sKJrerTv1Njn6sSkbIo0KvK2ndgdrFRhRNvhIET/amnDK9+s4X/mbuKwiLDpz16RS+NKCgS5hToVeXrSdCoA5xxj7dcqwH0GBZ2c31+sWEPD85bzeCuzbi4d2sAalWvxmV92vhcmYiUR4FeFXathK1fw4WPQuINfldTqo3ph7n11SV0a16P5649JehJIUQkPCjQq8LiFyC+NvT7pd+VsPvgERas3ElB4dHD0b+yaAvV46oxZWyiwlwkAinQK1v2fljxOpx8DdT290PFjKw8Rk9exKY9mSU+X79mPC/d2L/CM/yISHhQoFe2Za9Cfjb0v9nXMvIKCrnl1SWk7s9m5k0D6dXu6LHGa8ZXo2Z8nA/ViUgoKNArU2EhJE2B9qdB65N9K8M5x4PzVvPVxr38/Zo+DO6mscZFopECPRTWLfAmbC4+1Wp+Duz/Hs79oy9l/WDal5t5bfFWbhnSlauKzTIkItFDgR4Ki56FvSnQpu/P19esD31Ge80TffLJujQe/c8aLjqpJb+70N8hBESkcinQj1fOYdi6CAbd6k0+EUbW7zrEHa8to0frBjw5su+PEzqLSHTSOKjHa/PnUJgH3cKr+/6ewznc+FISdWrEMWVsInVq6P9ukWin3/LjlfIhVK/rffAZJo7kFTBxxhL2HM7h9YmDaN2wtt8liUgVUKAfr5QPofNZvk4+UZRzjvvfWsmSLfuZNOYU+rRv5HdJIlJFdMvleOzd6E1C0e08vyv50bOfbuTtZdu554LuXHJya7/LEZEqpEA/Hikfed/D5P75+6t28reF6xnetw13nNvN73JEpIop0I9HyofQpAs06ex3JaxMzeDXs5fTr0MjHrvqZA11KxKDFOjHKj/Ha+ESBlfnuzKOcNPLSTStW5PJ1yVSq7q674vEIgX6sdr6NeRl+R7o2bkF3PxyMoeP5DNlbCLN69f0tR4R8Y9auZTFOfjw/8GmT49+LjMd4mpApzOquqofFRY6fjNnOat2ZPDCdYn0aN3At1pExH9BXaGb2VAzW29mKWZ2XwnPn2VmS80s38yuDn2ZPvn8Cfjyn1C9NtRr+fOvlr3gnAegRl3fynvyw+9YsHIXD/yiB+f3bOlbHSISHsq9QjezOGAScAGQCiSZ2Xzn3Joim20FxgG/rYwifbF6Lnz8v9B7BFw5Oeymipu7bDvPfJzCyMT23HSm/x/Kioj/grnlMgBIcc5tAjCzWcBw4MdAd85tDjxXWAk1Vo2cw5CZ5j3evwXe/hW0HwjDngm7MF+yZT/3vrmCgZ2b8MjlmrxZRDzBBHpbYFuR5VRgYOWU45O9G2HqBZC196d1DTvAyFehei3/6ipB6v4sJs5Ipk3DWjx/7anUiNfn2iLiCSbQS7r8O3pCymB2ZDYBmADQoUOHY9lF6GXvh5kjvMfD/gVxgbk0uwyBes39qqpEh47kMf6lZHLyC5k1oT+N64bHcAMiEh6CCfRUoH2R5XbAjmN5M+fcZGAyQGJi4jH9pxBSBXnw+vXeLZax86HjYL8rKlVBoeOuWctJST/M9BsG0K1FPb9LEpEwE0ygJwEJZtYZ2A6MAsZUalWhVJAPy2b8/HbKD3Ysg+//C5c/F3Zh7pxj3vIdbD+QDcCaHQf5eF0aj1zeizMSNIWciByt3EB3zuWb2e3AQiAOmOacW21mDwPJzrn5ZtYfeBtoDFxmZg85506q1MqD9f7vvXk9S2Rw9n3QN/z+f3r+s0089v66n62beFYXrjuto08ViUi4C6pjkXNuAbCg2LoHizxOwrsVE16+meyF+eA74NwHj37e7Kd75mHk/VW7eHzhOi7r04YnrjkZwzCD6nH6AFREShe9PUVTPvSuzk+4GM5/CKpFxvgmq7ZncPfs5fRp14i/XX0yNeMjo24R8V90Bnr6ephzA7Q4Ca58IazDfMveTH7/5goOZOUBsP1ANo3rVGfy9adqkC0RqZDoC/TMvV4zxPhaMPo1qBm+rUEysvO48aUk9hzOZWDnJgAktKzP7ed0o0X98Gr/LiLhL7oCPT8HZl8LB3fCDQugUfvyX+OT/IJCbp+5lK37spgxfiCndWnqd0kiEuGiJ9Cdg3fvhq1fwVVToV1iSHZ7JK+A1P3ZIdlXUS9++T2fb9jD41edrDAXkZCInkD/8ilY/qrXDLF3aAZ83JeZy5XPfsnmvVkh2V9xE87qwoj+4ftXhIhElugI9LXveuOWn3QlDDlqdN9jkpNfwK9mLGFHxhEevaIX9WuFtnlj/VrxnJ0QXkMLiEhki/xA37kC3roZ2p4Clz8bkpERnXP84e1VLN68j6dH91sqRQMAAAUhSURBVGNYnzYhKFREpHJFXqBvXfTzGYSWvgy1G8Oomd5EFEFyzjF3+Xa2lHA7Zdu+bN5cmspd5yUozEUkYkReoG/7Bj79y0/LdZvDtW9B/VYV2s2Uz7/n0QVrS31+RGI7fn1+wrFWKSJS5SIv0Aff6X0VVcHbLB+u2c2f31vLxb1b8czoU6hWwss1aYSIRJrIC/TjDNq1Ow9y16xl9G7bkL9f05e4ktJcRCQCRVygv560jRc+33TMr9918Aj1a1XnhesTqV1DXetFJHpEXKA3qlOdhJbH3p2/Z5sG3DKkKy0bqGu9iESXiAv0C09qxYUnVewDUBGRWKABtkVEooQCXUQkSijQRUSihAJdRCRKKNBFRKKEAl1EJEoo0EVEooQCXUQkSphzzp83NksHthzjy5sBe0JYTqSIxeOOxWOG2DzuWDxmqPhxd3TOlTg7jm+BfjzMLNk5F5pJQyNILB53LB4zxOZxx+IxQ2iPW7dcRESihAJdRCRKRGqgT/a7AJ/E4nHH4jFDbB53LB4zhPC4I/IeuoiIHC1Sr9BFRKQYBbqISJSIuEA3s6Fmtt7MUszsPr/rqQxm1t7MPjGztWa22szuCqxvYmYfmNmGwPfGftcaamYWZ2bLzOzdwHJnM/smcMyzzayG3zWGmpk1MrM3zGxd4JwPipFzfXfg53uVmb1mZrWi7Xyb2TQzSzOzVUXWlXhuzfN0INtWmNkpFX2/iAp0M4sDJgG/AHoCo82sp79VVYp84DfOuR7AacBtgeO8D/jIOZcAfBRYjjZ3AWuLLD8GPBk45v3AeF+qqlxPAe87504E+uAdf1SfazNrC9wJJDrnegFxwCii73y/BAwttq60c/sLICHwNQF4rqJvFlGBDgwAUpxzm5xzucAsYLjPNYWcc26nc25p4PEhvF/wtnjHOj2w2XTgcn8qrBxm1g64BJgSWDbgXOCNwCbReMwNgLOAqQDOuVzn3AGi/FwHxAO1zSweqAPsJMrOt3Puv8C+YqtLO7fDgZedZxHQyMxaV+T9Ii3Q2wLbiiynBtZFLTPrBPQDvgFaOud2ghf6QAv/KqsU/wTuBQoDy02BA865/MByNJ7vLkA68GLgVtMUM6tLlJ9r59x24AlgK16QZwBLiP7zDaWf2+POt0gLdCthXdS2uzSzesCbwK+dcwf9rqcymdmlQJpzbknR1SVsGm3nOx44BXjOOdcPyCTKbq+UJHDfeDjQGWgD1MW75VBctJ3vshz3z3ukBXoq0L7Icjtgh0+1VCozq44X5q86594KrN79w59gge9pftVXCU4HhpnZZrxbaefiXbE3CvxJDtF5vlOBVOfcN4HlN/ACPprPNcD5wPfOuXTnXB7wFjCY6D/fUPq5Pe58i7RATwISAp+E18D7EGW+zzWFXODe8VRgrXPuH0Wemg+MDTweC8yr6toqi3PufudcO+dcJ7zz+rFz7pfAJ8DVgc2i6pgBnHO7gG1mdkJg1XnAGqL4XAdsBU4zszqBn/cfjjuqz3dAaed2PnB9oLXLaUDGD7dmguaci6gv4GLgO2Aj8Ae/66mkYzwD70+tFcDywNfFePeUPwI2BL438bvWSjr+IcC7gcddgMVACjAHqOl3fZVwvH2B5MD5ngs0joVzDTwErANWATOAmtF2voHX8D4jyMO7Ah9f2rnFu+UyKZBtK/FaAFXo/dT1X0QkSkTaLRcRESmFAl1EJEoo0EVEooQCXUQkSijQRUSihAJdRCRKKNBFRKLE/wehLee846ZqtQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "理论上，对于Linear Unit来说，cross entropy和MSE是等价的，但实际上从大多数跑出来的结果看，cross entropy都要优于MSE。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
